Merge remote-tracking branch 'origin/master' into debug-next
diff --git a/.gitignore b/.gitignore index f0351b8..f3efefe 100644 --- a/.gitignore +++ b/.gitignore
@@ -73,6 +73,7 @@ /vmlinux.symvers /vmlinux.thinlto-index /vmlinux.unstripped +/vmlinux.BTF.inline /vmlinux-gdb.py /vmlinuz /System.map
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 1b2ae71..03636d799 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -657,6 +657,22 @@ See Documentation/admin-guide/bootconfig.rst + bpf.keyring_unsealed= + [BPF] When set to 1, leave the bpf keyring unsealed + after initialization so that userspace can provision + keys. Once the keyring is restricted it becomes active + and can be used for BPF program signature verification. + + Setting this also means that the bpf keyring is the + only non-system keyring a loader may select for the + rest of the boot: caller-supplied user/session + keyrings are refused with -EPERM, whether or not + provisioning actually completed. The system keyrings + stay selectable. Leaving it unset keeps the prior + behaviour, where a caller-supplied keyring is allowed. + + See Documentation/bpf/signing.rst + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. @@ -3130,6 +3146,32 @@ kvm.enable_vmware_backdoor=[KVM] Support VMware backdoor PV interface. Default is false (don't support). + kvm.gmem_in_place_conversion= + [KVM] Controls whether KVM enables in-place conversion + support for guest_memfd and tracks the private/shared + state of memory per guest_memfd instead of per VM. + + If enabled, KVM enables the KVM_SET_MEMORY_ATTRIBUTES2 + ioctl on guest_memfd file descriptors and disables the + legacy VM-scoped KVM_SET_MEMORY_ATTRIBUTES ioctl for + private memory state tracking. Only the + KVM_MEMORY_ATTRIBUTE_PRIVATE attribute moves to + per-guest_memfd tracking; other attributes remain + per-VM. + + This parameter toggles KVM's in-place conversion + capability support. Whether a VMM uses separate backends + or out-of-place memory management is determined by + userspace VMM design. + + Note, this parameter is only available when + CONFIG_KVM_PRIVATE_MEMATTR_VM=y. When + CONFIG_KVM_PRIVATE_MEMATTR_VM is not set, in-place + conversion is enabled (and cannot be disabled) if KVM + is built with private memory support. + + Default is N (off). + kvm.nx_huge_pages= [KVM] Controls the software workaround for the X86_BUG_ITLB_MULTIHIT bug.
diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index d782250..afe109a 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -276,7 +276,8 @@ +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU L1 | #3878312 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU S3 | #3995052 | N/A | +| ARM | MMU S3 | #3995052, | N/A | +| | | #3673557 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | GIC-700 | #2941627 | ARM64_ERRATUM_2941627 | +----------------+-----------------+-----------------+-----------------------------+
diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst index eb19c94..98d70b8 100644 --- a/Documentation/bpf/bpf_design_QA.rst +++ b/Documentation/bpf/bpf_design_QA.rst
@@ -221,9 +221,14 @@ Q: How much stack space a BPF program uses? ------------------------------------------- -A: Currently all program types are limited to 512 bytes of stack -space, but the verifier computes the actual amount of stack used -and both interpreter and most JITed code consume necessary amount. +A: A program may use up to 2 KiB of stack, combined over its call +chain, when the JIT of the architecture reports support for large +stacks (currently x86-64 and arm64); a single function may use all of +it, and every frame of a program running on a private stack gets the +whole amount. Elsewhere, and for programs verified for the interpreter, +the limit is 512 bytes; the interpreter itself never runs a frame +larger than that. The verifier computes the actual amount of stack +used and both interpreter and most JITed code consume necessary amount. Q: Can BPF be offloaded to HW? ------------------------------
diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst index edf8107..4729efd 100644 --- a/Documentation/bpf/bpf_devel_QA.rst +++ b/Documentation/bpf/bpf_devel_QA.rst
@@ -374,9 +374,10 @@ Please refer to the git log (``arch/*/net/``) to locate the necessary people for helping out. -Also always make sure to add BPF test cases (e.g. test_bpf.c and -test_verifier.c) for new instructions, so that they can receive -broad test coverage and help run-time testing the various BPF JITs. +Also always make sure to add BPF test cases (e.g. the jit_insns test +in test_progs and test_verifier.c) for new instructions, so that they +can receive broad test coverage and help run-time testing the various +BPF JITs. In case of new BPF instructions, once the changes have been accepted into the Linux kernel, please implement support into LLVM's BPF back @@ -514,10 +515,11 @@ from that kernel ``xyz`` as well. Do not expect that the BPF selftest from the latest mainline tree will pass all the time. -In particular, test_bpf.c and test_verifier.c have a large number of -test cases and are constantly updated with new BPF test sequences, or -existing ones are adapted to verifier changes e.g. due to verifier -becoming smarter and being able to better track certain things. +In particular, test_verifier.c and the instruction tests in test_progs +have a large number of test cases and are constantly updated with new +BPF test sequences, or existing ones are adapted to verifier changes +e.g. due to verifier becoming smarter and being able to better track +certain things. LLVM ====
diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index 3f05f17..29de122 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst
@@ -88,6 +88,9 @@ #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ #define BTF_KIND_ENUM64 19 /* Enumeration up to 64-bit values */ + #define BTF_KIND_LOC_PARAM 20 /* Location description (register, const etc) */ + #define BTF_KIND_LOC_PROTO 21 /* Set of location parameters for site */ + #define BTF_KIND_LOCSEC 22 /* Section with site descriptions */ Note that the type section encodes debug info, not just pure types. ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram. @@ -104,11 +107,13 @@ * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64. + /* "size" is used by INT, ENUM, STRUCT, UNION, ENUM64 and + * LOC_PARAM. * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. + * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. It is unused by + * LOC_PROTO and LOCSEC. * "type" is a type_id referring to another type. */ union { @@ -416,7 +421,7 @@ * ``info.vlen``: 0 * ``type``: the type of the variable -``btf_type`` is followed by a single ``struct btf_variable`` with the +``btf_type`` is followed by a single ``struct btf_var`` with the following data:: struct btf_var { @@ -563,6 +568,80 @@ If the original enum value is signed and the size is less than 8, that value will be sign extended into 8 bytes. +2.2.20 BTF_KIND_LOC_PARAM +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: 0 + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOC_PARAM + * ``info.vlen``: number of 32-bit location value words + * ``size``: size in bytes of the represented parameter: 1 through 16 + +``btf_type`` is followed by a ``struct btf_loc_param`` and ``info.vlen`` +number of 32-bit value words.:: + + struct btf_loc_param { + __u32 flags; + __u32 values[]; + }; + +The ``flags`` field describes how to interpret ``values``: + + * ``BTF_LOC_PARAM_CONST`` describes a constant; the value is stored in + low-word, high-word order when it requires 64 bits. + * ``BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST`` describes an address offset + relative to the runtime base address of the kernel or module image. + * ``BTF_LOC_PARAM_REG`` with one word describes a register number; with two + words it describes a multi-register parameter. + * ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET`` describes an address held in + a register plus an offset. Adding ``BTF_LOC_PARAM_DEREF`` dereferences + that address. ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF`` with one word + dereferences the value held in the register. + * ``BTF_LOC_PARAM_SIGNED`` makes a constant or offset signed. A constant's + signed width is ``size``. For a register-relative offset, its signed + width is the number of offset value words times 32 bits. + +2.2.21 BTF_KIND_LOC_PROTO +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: 0 + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOC_PROTO + * ``info.vlen``: number of function parameter locations + * ``type``: 0 + +``btf_type`` is followed by ``info.vlen`` number of ``__u32`` BTF type IDs. +Each entry corresponds to a function parameter at an inline site. An entry is +either 0, meaning that no location information is available, or the type ID +of a ``BTF_KIND_LOC_PARAM``. + +2.2.22 BTF_KIND_LOCSEC +~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: offset to a valid ELF section name + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOCSEC + * ``info.vlen``: number of inline sites in the section + * ``type``: 0 + +``btf_type`` is followed by ``info.vlen`` number of ``struct btf_loc``.:: + + struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; + }; + +The ``func`` field is the non-zero type ID of the ``BTF_KIND_FUNC`` being +described. ``loc_proto`` is the non-zero type ID of the associated +``BTF_KIND_LOC_PROTO``. ``offset`` is the location address offset relative to +the runtime base address of the ELF section associated with the LOCSEC. +For example, a LOCSEC named ``inline.text`` contains records for ``.text`` +whose offsets are relative to the runtime base of that section. + 2.3 Constant Values -------------------
diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst index 2c872a1..3ccc7b0 100644 --- a/Documentation/bpf/clang-notes.rst +++ b/Documentation/bpf/clang-notes.rst
@@ -23,8 +23,11 @@ Jump instructions ================= -If ``-O0`` is used, Clang will generate the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) -instruction, which is not supported by the Linux kernel verifier. +Clang generates the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) instruction for calls +through a function pointer. The Linux kernel verifier accepts it only when it +can prove that the register holds the address of a static BPF function, see +Documentation/bpf/linux-notes.rst. If ``-O0`` is used, Clang will generate this +instruction for helper calls as well, which is not supported. Atomic operations =================
diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 6691fe8..73578a4 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst
@@ -164,19 +164,32 @@ 2.3.3 __uninit Annotation ------------------------- -This annotation is used to indicate that the argument will be treated as -uninitialized. +Use ``__uninit`` on a pointer parameter for an output buffer whose incoming +contents the kfunc does not read. -An example is given below:: +For generic memory buffers, the kfunc may leave bytes untouched, including +on error returns. The writable range is determined by the pointed-to type +or the associated ``__sz`` or ``__szk`` size argument. - __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr_kern *ptr__uninit) +The annotation does not change the accepted pointer types. A stack-backed +struct passed as a generic memory buffer must still be scalar-only. + +Generic stack buffers may be uninitialized before the call. The call does +not make previously uninitialized bytes readable unless the program is +allowed to read uninitialized stack memory (normally requiring +``CAP_PERFMON``). Other callers must initialize those bytes themselves +before reading them. Stack bounds and variable-offset restrictions still +apply. + +For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a +dynptr in the supplied storage. For example:: + + __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr *ptr__uninit) { ... } -Here, the dynptr will be treated as an uninitialized dynptr. Without this -annotation, the verifier will reject the program if the dynptr passed in is -not initialized. +Without this annotation, a dynptr argument must already be initialized. 2.3.4 __nullable Annotation --------------------------- @@ -315,6 +328,29 @@ non-NULL before use, in-line with existing semantics of arena pointers used in a program (or obtained from any other source). +2.3.9 __ctx_out Annotation +-------------------------- + +This annotation is used to indicate that the argument is an output +parameter of the attached hook, passed through from the program's +context. The verifier requires the register to be a trusted read-only +pointer to fixed-size memory, which can only be produced by loading an +argument described by the program's ctx_arg_info from the context. The +program itself cannot write through the pointer; the kfunc may. + +An example is given below:: + + __bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs, + int *xattr_count__ctx_out, + ...) + { + ... + } + +In this case, a program attached to the ``inode_init_security`` LSM hook +can pass the hook's own ``xattr_count`` argument through to the kfunc, +which claims xattr slots by writing through it. + .. _BPF_kfunc_nodef: 2.4 Using an existing kernel function @@ -546,7 +582,7 @@ The verifier would allow a ``struct bpf_cpumask *`` to be passed to a kfunc taking a ``cpumask_t *`` (which is a typedef of ``struct cpumask *``). For -instance, both ``struct cpumask *`` and ``struct bpf_cpmuask *`` can be passed +instance, both ``struct cpumask *`` and ``struct bpf_cpumask *`` can be passed to bpf_cpumask_test_cpu(). In some cases, this type-aliasing behavior is not desired. ``struct @@ -585,6 +621,80 @@ therefore access up to ``GUARD_SZ / 2`` past it without bounds-checking against the arena. Larger accesses must verify the range explicitly. +2.9 kfunc Return Values +----------------------- + +A kfunc may return a scalar, a pointer, or a small struct or union by +value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. + +A struct or union returned by value must be composed only of scalars and arena +pointers, where a scalar is an integer or an enum and an arena pointer is one +carrying the ``btf_type_tag("arena")`` attribute. Those may be nested in +structs and unions and in arrays of any number of dimensions, in any +combination, as long as what the nesting bottoms out in is a scalar or an arena +pointer. Its bytes are handed back to the program as the raw contents of R0 +(and R2), so a member of any other pointer type would be laundered into a +scalar and escape the verifier's pointer provenance and reference tracking. A +struct or union with such a member is therefore rejected at load time, and so +is one with a floating-point member, which the ABI may not return in R0:R2 at +all. + +An arena pointer member is handed back as a scalar too, but nothing is lost by +that. The program must ``cast_kern()`` the value before it can be used, and the +verifier allows that cast on any scalar, so a laundered arena address gives the +program no reach it did not already have. The result is confined to the +program's arena in either case. + +A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a +struct or union of scalars and arena pointers, or an ``__int128``. Such a +value is returned in the register pair R0:R2, matching the convention LLVM +uses for the BPF target: the first 8 bytes in R0 and the second 8 bytes in R2. +A struct or union of 8 bytes or less is returned in R0 alone. + +:: + + struct bpf_pair { __u64 a, b; }; /* 16 bytes */ + + __bpf_kfunc struct bpf_pair bpf_kfunc_get_pair(void) + { + struct bpf_pair p = { .a = 1, .b = 2 }; + + return p; /* p.a in R0, p.b in R2 */ + } + +Returning a value in the R0:R2 pair requires the JIT to place the second +half of the return value into R2, which not every architecture supports +right now. A kfunc with a return value larger than 8 bytes is therefore +rejected at load time on a JIT that does not advertise this capability (see +``bpf_jit_supports_kfunc_ret_reg_pair()``), and such a program is never run +by the interpreter. A return value larger than 16 bytes is not supported. + +The same R0:R2 convention applies to a BPF subprogram, global or static, that +returns an ``__int128`` or a struct or union larger than 8 bytes. It is only +used when the program is JITed, since the interpreter propagates only R0 out of +a subprogram: without a JIT the return value stays in R0 alone, and a caller +reading R2 is rejected for reading an uninitialized register. A global +subprogram is verified in isolation, so its by-value struct or union return is +restricted to scalars and arena pointers just like a kfunc's; a static +subprogram is verified inline and has no such restriction. The main program is +not covered: its return value is the program's exit code, read out of R0 +alone, so a declared upper half is never looked at. + +A global subprogram must leave a scalar in *every* register of the pair, so +both halves of the returned value have to be assigned. Leaving the upper half +uninitialized is not merely untidy: the compiler is then free to leave R2 +holding whatever it happened to hold, which for a subprogram taking a pointer +argument is typically that pointer. Handing the caller an unknown scalar built +from a pointer is a leak, so the verifier rejects it with:: + + At subprogram exit the register R2 is not a scalar value (...) + +Initialize the whole return value, for example ``struct pair p = {};``, to +avoid this. A static subprogram is exempt from the scalar-only rule: it is +verified inline, so an unassigned R2 is simply passed back to the caller as +uninitialized and only a caller that reads it fails. A stack pointer left in +R2 is still rejected there, just as one in R0 is. + .. _BPF_kfunc_lifecycle_expectations: 3. kfunc lifecycle expectations
diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst index 00d2693..6c036b5 100644 --- a/Documentation/bpf/linux-notes.rst +++ b/Documentation/bpf/linux-notes.rst
@@ -15,10 +15,33 @@ Jump instructions ================= -``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), where the helper function -integer would be read from a specified register, is not currently supported -by the verifier. Any programs with this instruction will fail to load -until such support is added. +``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), ``callx dst``, performs an indirect +call of a BPF function whose address is held in the ``dst`` register. The +``src``, ``offset`` and ``imm`` fields are reserved and must be zero. + +The address of a BPF function gets into a register in one of two ways: + +* it is loaded by a 64-bit immediate instruction with ``src`` = + ``BPF_PSEUDO_FUNC``; +* it is read, with a 64-bit load, from a frozen read-only array map, that no + other program uses, that holds its read-only data: tables of functions, + structures of operations, vtables, where pointers to functions may be mixed + with other data. In the map a pointer to a function is the offset in bytes + of its first instruction in the program, and that is how the verifier + recognizes it. It is replaced with the address of the function when + the program is loaded. The program reads it from there, which requires + ``CAP_PERFMON``. + +In both cases only static functions can be referenced. Therefore all functions +that can be called indirectly are known to the verifier before it starts to +analyze the program, and ``callx`` is verified as a direct call of every +function that ``dst`` may point to at that instruction. The same rules apply: +the calls can not be recursive, and the depth of the call chain and its +combined stack size are limited. + +Calling helper or kernel functions through a register, indirect calls of global +functions, and tail calls in functions that are called via ``callx`` are not +supported. ``callx`` requires the BPF JIT. Maps ====
diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst index e73eaae..27fdad3 100644 --- a/Documentation/bpf/signing.rst +++ b/Documentation/bpf/signing.rst
@@ -254,21 +254,24 @@ (``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning: validly signed *and* fully verified. -A more realistic admission policy than "is it signed at all": accept programs -signed by a system keyring, accept a user-keyring signature only if the -key/keyring it was verified against is on an explicit allowlist, and emit a -tamper-evident record of every decision so that even denied attempts are -auditable. (Illustrative - error checking elided.) +A more realistic admission policy than "is it signed at all": base trust in +the bpf keyring or an explicit allowlist of a caller-supplied staging key/ +keyring, and emit a record of every decision including denied attempts. +(Illustrative - error checking elided.) .. code-block:: c - /* Serials of user keys/keyrings we additionally trust. */ + /* + * Serials of caller-supplied keyrings we are willing to stage. Empty + * on a system that has committed to the bpf keyring, where the kernel + * refuses them anyway. + */ struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, __s32); /* keyring_serial */ __type(value, __u8); __uint(max_entries, 64); - } trusted_user_keys SEC(".maps"); + } staging_keys SEC(".maps"); /* Audit stream consumed by a userspace logger. */ struct { @@ -291,11 +294,18 @@ if (kernel) return 0; /* trust in-kernel loads */ - if (verdict != BPF_SIG_VERIFIED) + if (verdict != BPF_SIG_VERIFIED) { ret = -EPERM; /* must be validly signed */ - else if (ktype == BPF_SIG_KEYRING_USER && - !bpf_map_lookup_elem(&trusted_user_keys, &serial)) - ret = -EPERM; /* key/keyring not allowlisted */ + } else switch (ktype) { + case BPF_SIG_KEYRING_BPF: + break; + case BPF_SIG_KEYRING_USER: + if (!bpf_map_lookup_elem(&staging_keys, &serial)) + ret = -EPERM; + break; + default: + ret = -EPERM; /* keyring not in policy */ + } d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0); if (d) { @@ -309,6 +319,11 @@ return ret; } +Such a policy is what makes a caller-supplied keyring usable at all on a system +that does not boot with ``bpf.keyring_unsealed=1``: the allowlist bounds which +staged keys count, and the LSM itself has to protect them from being tampered +with. + Observing a verified load: ``security_bpf_prog()`` -------------------------------------------------- @@ -381,8 +396,9 @@ a deny-by-default admission policy needs no second enforcement point. Use ``security_bpf_prog()`` to record or finally gate the verified programs once they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields -let a policy distinguish, for example, "verified and signed by a builtin key" -from "verified by a user key". A policy LSM such as IPE could consume the same +let a policy distinguish "verified against the operator's bpf keyring" from +"verified against a keyring the loader supplied itself", which is the +distinction that matters most. A policy LSM such as IPE could consume the same hooks to enforce system policy without writing any BPF, though none implements this today. @@ -390,33 +406,158 @@ ======== ``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified -against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING`` -and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings; -any other value is treated as the serial of a user/session key or keyring. -The keyring is looked up first, before the signature bytes are examined, so a -signature naming a non-existent keyring is rejected up front, and a failed -verification aborts the load - so a program that loads successfully with a -signature always has consistent keyring fields recorded. +against. Four values are well-known; anything else is resolved as a caller- +supplied key or keyring, named either by its serial or by one of the other +``KEY_SPEC_*`` ids: + +.. list-table:: + :header-rows: 1 + + * - ``keyring_id`` + - Keyring + * - ``0`` + - builtin trusted keyring + * - ``VERIFY_USE_SECONDARY_KEYRING`` (``1``) + - secondary trusted keyring + * - ``VERIFY_USE_PLATFORM_KEYRING`` (``2``) + - platform keyring + * - ``KEY_SPEC_BPF_KEYRING`` (``-9``) + - the bpf keyring + * - anything else + - a caller-supplied user/session key or keyring + +The keyring is resolved first, before the signature bytes are examined, so a +signature naming a keyring that cannot be used is rejected up front, and a +failed verification aborts the load - a program that loads successfully with +a signature therefore always has consistent keyring fields recorded. + +The bpf keyring +--------------- + +A system keyring needs a kernel rebuild or a vouched-for enrollment to rotate a +key, and grants BPF-signing trust to keys trusted for everything else in the +kernel too. A caller-supplied keyring, at the other extreme, is filled by the +very process that loads the program and so carries no trust of its own. + +The bpf keyring fills that gap and is the trust anchor which a signed BPF +deployment should be built on top of: a keyring named ``.bpf``, selected with +``KEY_SPEC_BPF_KEYRING``, that an operator provisions at boot with a key +scoped to BPF program loading and nothing else in the kernel's trust hierarchy. +It is owned by the operator rather than by the loader, and rotatable across a +reboot without touching the kernel image. It is modelled after the dm-verity +keyring (see ``dm_verity.keyring_unsealed=``) and provisioned the same way: an +initrd runs the ``keyctl`` steps below before handing off to the rootfs. + +Provisioning +~~~~~~~~~~~~ + +The keyring is created during ``late_initcall`` and is **sealed empty** by +default: it carries a reject-all restriction, so no key can ever be added and +``KEY_SPEC_BPF_KEYRING`` fails with ``-ENOKEY`` for the whole boot. + +``bpf.keyring_unsealed=1`` leaves it unrestricted at init so the initrd can +provision it. The keyring is not linked into any process keyring, but the +``KEY_SPEC_BPF_KEYRING`` special key id addresses it directly, so its serial +does not have to be looked up first. Steps would be as follows:: + + keyctl padd asymmetric "" -9 < signing_key.der + keyctl restrict_keyring -9 + +Both steps are required: the keyring is consulted only once it is **non-empty +and restricted**. An unrestricted keyring is ignored even when it holds keys, +so a half-provisioned keyring is inert rather than a weaker trust anchor, and a +load naming it fails with ``-ENOKEY`` and a verifier log. Restricting cannot +be undone. + +More than one key is enrolled by repeating the ``keyctl padd`` step; the +restriction is applied once, after the last of them:: + + for key in /etc/bpf/keys/*.der; do + keyctl padd asymmetric "" -9 < $key + done + + keyctl restrict_keyring -9 + keyctl show -9 + +The restriction bounds what can be added, never what can be taken away. A key +that is already enrolled can still be unlinked, and the keyring cleared or +revoked, by anything running as root. That does not weaken the anchor, since +a keyring left empty is no longer consulted and a load naming it fails with +``-ENOKEY``, but it does take signed loading out until the next boot. Dropping +the user permissions the keyring no longer needs would close that; as a third +step in the initrd:: + + keyctl setperm -9 0x08030000 + +What remains is ``KEY_POS_SEARCH`` for the in-kernel search during verification, +plus ``KEY_USR_VIEW`` and ``KEY_USR_READ`` so the keyring stays visible in +``/proc/keys``. ``KEY_SPEC_BPF_KEYRING`` names the keyring but conveys no rights +of its own - the keyring is not possessed by anyone, so the ``KEY_POS_*`` bits +are only ever reached by the in-kernel search, and userspace is left with what +the ``KEY_USR_*`` bits grant. Once they are dropped, addressing it by the serial +``/proc/keys`` reports gets no further than the special id does. + +Provisioning has to complete before control passes to the rootfs. The keyring +takes any number of keys for as long as it is unrestricted, so it is the +restriction that bounds the enrolled set, not the first enrollment. Before +the initrd hands off control, it must therefore restrict the keyring after +the last enrollment. + +Enforcement +~~~~~~~~~~~ + +``bpf.keyring_unsealed=1`` states that the bpf keyring is *the* trust anchor for +this boot, so it does more than unseal. From the first program load onwards a +caller-supplied user/session keyring is refused with ``-EPERM`` and a verifier +log message, whether or not provisioning ever completed. The system keyrings +stay selectable. + +Enforcement is readable at ``/sys/module/bpf/parameters/keyring_unsealed``, and +read-only there: the flag is ``__ro_after_init`` behind a 0444 parameter. It is +also derived from the boot flag rather than from the keyring's runtime state, +so there is no window early in boot during which a caller-supplied keyring is +still accepted. + +Caller-supplied keyrings are for staging +---------------------------------------- + +A ``keyring_id`` naming a user or session key or keyring is a *staging* +mechanism, not a trust anchor: it is filled by the same userspace that loads the +program, so verifying against it establishes only that the loader signed what it +loaded. Its purpose is to let software installed onto a running system - whose +signing key is not enrolled anywhere yet - run signed until that key reaches the +bpf keyring on the next boot. + +A system that has committed to the bpf keyring refuses this path outright (see +`Enforcement`_). A system that has not can still allow it, but a policy must +never treat ``BPF_SIG_KEYRING_USER`` as equivalent to the bpf or system +keyrings; it should allowlist the specific serials it is willing to stage and +pair that with a BPF LSM policy protecting those keys from tampering, as in +`Enforcement via LSMs`_. + +Recorded fields +--------------- Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: ``keyring_type`` (``enum bpf_sig_keyring``) Classified purely from ``keyring_id`` whenever the program is signed: ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system - keyrings, or ``_USER`` for a user/session keyring. It is - ``BPF_SIG_KEYRING_NONE`` for an unsigned program. + keyrings, ``_BPF`` for the bpf keyring, or ``_USER`` for a caller-supplied + user/session keyring. It is ``BPF_SIG_KEYRING_NONE`` for an unsigned + program. ``keyring_serial`` (``s32``) Set **only** on a successful verification, to the serial of the - **user/session key or keyring** that ``keyring_id`` resolved to - the + **caller-supplied key or keyring** that ``keyring_id`` resolved to - the object the signature was verified against, not the individual asymmetric key inside it that matched the signer. Passing ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's - serial. The system keyrings are trusted as a whole and expose no serial - here, so the serial is ``0`` for builtin, secondary and platform - signatures, and ``0`` for unsigned programs. In other words, a non-zero - ``keyring_serial`` is exactly "verified against the user key/keyring with - this serial". + serial. The system keyrings and the bpf keyring are trusted as a whole and + expose no serial here, so the serial is ``0`` for them, and ``0`` for + unsigned programs. A non-zero ``keyring_serial`` is therefore exactly + "verified against the caller-supplied key/keyring with this serial", which + is exactly the case a policy has to scrutinise. .. list-table:: :header-rows: 1 @@ -436,16 +577,51 @@ * - ``VERIFY_USE_PLATFORM_KEYRING`` - ``BPF_SIG_KEYRING_PLATFORM`` - ``0`` - * - other (a user/session key serial) + * - ``KEY_SPEC_BPF_KEYRING`` + - ``BPF_SIG_KEYRING_BPF`` + - ``0`` + * - other (a caller-supplied key serial) - ``BPF_SIG_KEYRING_USER`` - serial of the resolved key/keyring -Producing a signed object -========================== +Producing and loading a signed object +===================================== + +Generating a signing key +------------------------ + +Signing is algorithm agnostic: the algorithm comes from the X.509 certificate +and the PKCS#7 ``SignerInfo``. Anything the X.509 and PKCS#7 parsers understand +works with no BPF-side change. Both examples below read the certificate request +config from ``x509.genkey``, for which ``certs/x509.genkey`` serves as a +template (see Documentation/admin-guide/module-signing.rst). RSA:: + + openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \ + -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +ML-DSA-87 (FIPS-204), which needs openssl 3.5 or later, and both +``CONFIG_CRYPTO_MLDSA`` and ``CONFIG_CRYPTO_SHA512`` in the kernel - the latter +for the signedAttrs digest described below, which ``CONFIG_CRYPTO_MLDSA`` does +not select. Note the absence of a digest option: ML-DSA hashes the message +itself, so openssl ignores an explicit digest for it:: + + openssl req -new -nodes -utf8 -days 36500 -batch -x509 \ + -newkey ML-DSA-87 -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +``bpftool`` handles the following internally: openssl before 4.0 cannot combine +ML-DSA with ``CMS_NOATTR``, so it falls back to signedAttrs, where only SHA-512 +is permitted. This mirrors what module signing does as well. + +Signing +------- ``bpftool`` generates and signs a light skeleton in one step:: - bpftool gen skeleton -L -S -k <private_key.pem> -i <certificate.x509> \ + bpftool gen skeleton -L -S -k signing_key.pem -i signing_key.der \ obj.bpf.o > obj.lskel.h ``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables @@ -454,12 +630,36 @@ reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader instructions so the metadata map can be bound to the loader. The signature and hash are embedded in the generated header; the certificate is used only for -signing and is not included. Loading the skeleton performs the -create/populate/freeze/load sequence described above. +signing and is not included. -At runtime the trusted public key must be present in the chosen keyring (for -example added to the session keyring, or built into the kernel's builtin trusted -keyring) for verification to succeed. +Loading +------- + +The generated skeleton exposes ``keyring_id``, which selects the keyring the +kernel verifies against. Set it between open and load; loading then performs +the create/populate/freeze/load sequence described above:: + + struct obj *skel = obj__open(); + + skel->keyring_id = KEY_SPEC_BPF_KEYRING; + err = obj__load(skel); + +For the staging case the same object is loaded against a keyring the caller +populated itself, which only works on a system that has not set +``bpf.keyring_unsealed=1``:: + + /* + * Staging only: this keyring is under the loader's own control and + * carries no trust of its own. See "Caller-supplied keyrings are for + * staging". + */ + key_id = add_key("asymmetric", "", der, der_sz, KEY_SPEC_SESSION_KEYRING); + skel->keyring_id = KEY_SPEC_SESSION_KEYRING; + err = obj__load(skel); + +Either way the trusted public key must already be in the chosen keyring for +verification to succeed. For the bpf keyring that enrollment happens once at +boot, see `Provisioning`_. UAPI reference ============== @@ -487,6 +687,13 @@ The map content is not hashed separately at all - it is covered, as bytes, by the program signature. +Kernel command line: + +``bpf.keyring_unsealed=`` + Set to ``1`` to leave the bpf keyring unsealed for provisioning, and to make + it the only non-system keyring a loader may select for the rest of the boot + (see `The bpf keyring`_). + Notes and limitations ====================== @@ -495,3 +702,5 @@ exceed it. - The metadata container is a single-element array map, accessed through ``map_direct_value_addr``. +- The bpf keyring needs ``CONFIG_KEYS``; without it there is no bpf keyring + and ``KEY_SPEC_BPF_KEYRING`` never resolves.
diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst index ba23a47..69d4ebe 100644 --- a/Documentation/core-api/dma-api.rst +++ b/Documentation/core-api/dma-api.rst
@@ -154,7 +154,7 @@ :: size_t - dma_opt_mapping_size(struct device *dev); + dma_max_opt_mapping_size(struct device *dev); Returns the maximum optimal size of a mapping for the device.
diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml index 9aca38a..8f8aae9 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml
@@ -35,6 +35,9 @@ minItems: 1 maxItems: 2 + iommus: + maxItems: 1 + required: - compatible - reg
diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 47ec7fa..6efbe73 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml
@@ -44,10 +44,12 @@ Optional since not all IOMMUs are attached to a clock gate. '#iommu-cells': - const: 1 + enum: [ 1, 5 ] description: - Has to be one. The single cell describes the stream id emitted by - a master to the IOMMU. + Has to be one or five. The first cell describes the stream id emitted by + a master to the IOMMU. The optional second and third cell describe the + aperture start of a master's DMA window and fourth and fifth cell describe + the size of DMA window. power-domains: maxItems: 1 @@ -66,11 +68,11 @@ compatible = "apple,t8103-dart"; reg = <0x82f80000 0x4000>; interrupts = <1 781 4>; - #iommu-cells = <1>; + #iommu-cells = <5>; }; master1 { - iommus = <&dart1 0>; + iommus = <&dart1 0 0x100 0x0 0x10 0x0>; }; - |+
diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml index 8295733..3c044e4 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml
@@ -83,6 +83,16 @@ register access with page 0 offsets. Set for Cavium ThunderX2 silicon that doesn't support SMMU page1 register space. + arm,instdata-override: + type: boolean + description: + Some SoCs like Google Tensor G5 (Laguna) have an SMMU integration + silicon erratum where AxPROT[2] signal is always 1, making all + transactions appear as instructions. + + This property enables a software workaround to override the + instruction/data attribute of incoming traffic to data. + allOf: - if: not:
diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index ffa33b7..6748351 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml
@@ -101,7 +101,9 @@ - qcom,glymur-smmu-500 - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 + - qcom,maili-smmu-500 - qcom,milos-smmu-500 + - qcom,nord-smmu-500 - qcom,qcm2290-smmu-500 - qcom,qcs615-smmu-500 - qcom,qcs8300-smmu-500 @@ -579,6 +581,8 @@ - qcom,glymur-smmu-500 - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 + - qcom,maili-smmu-500 + - qcom,nord-smmu-500 - qcom,sm8750-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500
diff --git a/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml new file mode 100644 index 0000000..5ca8ce4 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml
@@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom BCM2712 IOMMU + +maintainers: + - Daniel Drake <dan@reactivated.net> + - Florian Fainelli <florian.fainelli@broadcom.com> + - Jim Quinlan <james.quinlan@broadcom.com> + +description: + The BCM2712 SoC features multiple independent IOMMU instances providing + address translation for multimedia hardware blocks, such as the VC6 display + pipeline, camera receivers, and ISP. Each IOMMU has its own page tables, + control registers, and local TLB, and is additionally linked to a centralized + L2 TLB (IOMMUC). + +properties: + compatible: + const: brcm,bcm2712-iommu + + reg: + maxItems: 1 + + '#iommu-cells': + const: 0 + + brcm,iommu-cache: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle to the shared IOMMU cache (IOMMUC). Each IOMMU instance caches + translated descriptors in this centralized L2 TLB cache block and must + trigger cache invalidation commands on it when updating or unmapping page + table entries. + +required: + - compatible + - reg + - '#iommu-cells' + - brcm,iommu-cache + +additionalProperties: false + +examples: + - | + iommu@5200 { + compatible = "brcm,bcm2712-iommu"; + reg = <0x5200 0x80>; + brcm,iommu-cache = <&iommuc>; + #iommu-cells = <0>; + };
diff --git a/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml new file mode 100644 index 0000000..47c584a --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml
@@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommuc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom BCM2712 Shared IOMMU Cache (IOMMUC) + +maintainers: + - Daniel Drake <dan@reactivated.net> + - Florian Fainelli <florian.fainelli@broadcom.com> + - Jim Quinlan <james.quinlan@broadcom.com> + +description: + The BCM2712 IOMMUC is a centralized Translation Lookaside Buffer (TLB) cache + which accelerates address translation across the SoC's IOMMU devices. If an + address mapping is not found in the IOMMU's local TLB cache, then this + IOMMUC is consulted as a L2 TLB. The SoC includes a single IOMMUC which is + shared between all IOMMUs, and it must be explicitly invalidated when + modifying or unmapping IOMMU page tables. + +properties: + compatible: + const: brcm,bcm2712-iommuc + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + iommuc@5b00 { + compatible = "brcm,bcm2712-iommuc"; + reg = <0x5b00 0x80>; + };
diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index 26b7638..aaf5d76 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst
@@ -47,6 +47,7 @@ vfio-mediated-device vfio vfio-pci-device-specific-driver-acceptance + vfio-selftests Bus-level documentation =======================
diff --git a/Documentation/driver-api/vfio-selftests.rst b/Documentation/driver-api/vfio-selftests.rst new file mode 100644 index 0000000..9d39a22 --- /dev/null +++ b/Documentation/driver-api/vfio-selftests.rst
@@ -0,0 +1,252 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============== +VFIO Selftests +============== + +VFIO selftests are built on top of the kernel's selftests framework and are +located in ``tools/testing/selftests/vfio``. + +VFIO selftests enable kernel developers to write and run tests that take the +form of userspace programs that interact with VFIO and IOMMUFD uAPIs. VFIO +selftests can be used to write functional tests for new features, regression +tests for bugs, and performance tests for optimizations. + +These tests are designed to interact with real PCI devices, i.e. they do not +rely on mocking out or faking any behavior in the kernel. This allows the tests +to exercise not only VFIO but also IOMMUFD, the IOMMU driver, interrupt +remapping, IRQ handling, etc. + +Running Tests +============= + +Running VFIO selftests requires a device bound to the ``vfio-pci`` driver. +There are scripts in ``tools/testing/selftests/vfio/scripts/`` to help with +setting one up. + +Picking a Device +---------------- + +Some VFIO selftests require a device with a supported driver implementation in +the selftests library (see `Driver Framework`_), while other tests (such as +basic PCI configuration space or reset tests) can use any arbitrary PCI device. + +This section will be expanded in the future to guide developers on how to find +a device with a supported driver. In the meantime, the selftests library +includes driver support for the following devices: + +- Intel IGB Gigabit Ethernet controllers +- Intel IOAT (I/O Acceleration Technology) DMA engines +- Intel DSA (Data Streaming Accelerator) +- NVIDIA Falcon engines (found in many NVIDIA GPUs, e.g., GTX 1080 or RTX 2080) + +Note that using Intel DSA devices requires setting the module parameter +``vfio_pci.disable_denylist=Y`` to allow ``vfio-pci`` to bind to DSA devices. + +Setting up Devices +------------------ + +.. code-block:: sh + + tools/testing/selftests/vfio/scripts/setup.sh 0000:01:00.0 + +The script handles unbinding the device from its current driver and binding it +to ``vfio-pci``. Metadata about this device is stored in +``/tmp/vfio-selftests-devices`` so that the device can be cleaned up later +(bound back to its original driver). + +Execution +--------- + +VFIO selftests expect the PCI device BDF string as their final command-line +argument. e.g. + +.. code-block:: sh + + tools/testing/selftests/vfio/vfio_pci_device_test 0000:01:00.0 + +Alternatively, the PCI device BDF can be passed via the ``VFIO_SELFTESTS_BDF`` +environment variable, which can be useful when running all tests: + +.. code-block:: sh + + export VFIO_SELFTESTS_BDF=0000:01:00.0 + make -C tools/testing/selftests TARGETS=vfio install + tools/testing/selftests/kselftest_install/run_kselftest.sh -c vfio + +Cleanup +------- + +.. code-block:: sh + + tools/testing/selftests/vfio/scripts/cleanup.sh + +This script handles unbinding all previously setup devices from ``vfio-pci`` +and binding them back to their original driver. Specific devices can be cleaned +up by passing their BDF string to the cleanup script. + +Writing Tests +============= + +VFIO selftests leverage the ``kselftest_harness.h`` macros to structure setup +and teardown scenarios. A basic test involves: + +1. Declaring a test FIXTURE containing the device and IOMMU state. +2. Generating test variants that span the different IOMMU modes utilizing + ``FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES()``. +3. Initializing the device and binding the default driver ops. + +For a complete example of writing a VFIO selftest, refer to +``tools/testing/selftests/vfio/vfio_pci_device_test.c``. + +VFIO Selftests Library (libvfio) +================================ + +The selftests directory contains a helper library compiled from the ``lib/`` +subdirectory. It provides structures and helper functions to handle standard +operations like opening VFIO devices, creating VFIO containers, and interacting +with VFIO and IOMMUFD. + +All VFIO selftests have libvfio linked in by default. Tests can access the +library by including ``<libvfio.h>``. + +Core Objects +------------ + +The VFIO selftests framework revolves around two distinct core objects: +``struct iommu`` and ``struct vfio_pci_device``. It is important to understand +the distinction and relationship between them: + +- ``struct iommu`` represents an IOMMU domain and the API used to program it + (e.g. legacy VFIO Type1 vs IOMMUFD). It is responsible for managing the I/O + address space, handling DMA mappings, and translating between host virtual + addresses (HVA) and IO virtual addresses (IOVA). + +- ``struct vfio_pci_device`` represents a single PCI device. It encapsulates + the VFIO device file descriptor, handles access to the PCI configuration + space, manages memory-mapped BARs, and drives device interrupts. + +A typical test initializes an IOMMU instance first, then passes it when +initializing one or more PCI devices, linking the devices to the shared IOMMU +domain. + +``struct iommu`` +^^^^^^^^^^^^^^^^ + +The ``struct iommu`` abstracts away the complexity of managing VFIO containers, +IOMMU groups, and IOMMUFD contexts. A test allocates its IOMMU handle using +``iommu_init()`` passing one of the supported IOMMU modes: + +.. code-block:: c + + struct iommu *iommu = iommu_init(MODE_IOMMUFD); + ... + iommu_cleanup(iommu); + +The selftest framework is designed to work uniformly across all standard kernel +IOMMU APIs. The defined modes (in ``iommu.h``) include: + +- ``MODE_VFIO_TYPE1_IOMMU`` +- ``MODE_VFIO_TYPE1V2_IOMMU`` +- ``MODE_IOMMUFD_COMPAT_TYPE1`` +- ``MODE_IOMMUFD_COMPAT_TYPE1V2`` +- ``MODE_IOMMUFD`` + +VFIO selftests can replicate their test cases across all possible IOMMU modes +using FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES() to ensure kernel IOMMU APIs remain +compatible. + +``struct vfio_pci_device`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A PCI device is represented as a ``struct vfio_pci_device``. The device +representation is initialized using ``vfio_pci_device_init()``, which takes a +target BDF string, opens the device in VFIO, and binds it to an existing +``struct iommu`` instance: + +.. code-block:: c + + struct vfio_pci_device *device = vfio_pci_device_init(device_bdf, iommu); + ... + vfio_pci_device_cleanup(device); + +``struct iova_allocator`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To facilitate DMA mappings without hardcoding IOVAs (IO virtual addresses) that +might conflict with reserved platform address ranges, the library provides a +``struct iova_allocator``. + +The IOVA allocator queries the underlying IOMMU (via ``struct iommu``) for its +valid target IOVA ranges. Tests can safely carve out unique contiguous chunks +of IOVA space for device DMA by repeatedly calling: + +.. code-block:: c + + struct iova_allocator *allocator = iova_allocator_init(iommu); + iova_t addr = iova_allocator_alloc(allocator, size); + +Driver Framework +---------------- + +The primary goal of VFIO selftests is to verify the correctness of the kernel +code handling VFIO devices, including VFIO core, IOMMUFD, IOMMU drivers, page +pinning, DMA mapping/unmapping, interrupt remapping, and IRQ delivery. + +Because different physical devices have vastly different programming interfaces +for triggering work, the selftests library implements a standard driver +framework. This framework abstracts diverse hardware endpoints (such as Intel +IGB, IOAT, or DSA) behind a uniform interface, allowing tests to provoke a +broad range of devices to generate DMA operations and interrupts in a +standardized manner. + +By exercising DMA and interrupts with real endpoints, tests can thoroughly +stress and validate the underlying kernel paths, such as IOMMU page table +management, dirty page tracking, Device TLB and IOTLB invalidations, and +interrupt delivery under authentic hardware conditions. + +All endpoint driver implementations populate a common set of operations defined +in ``struct vfio_pci_driver_ops``: + +- ``probe()``: Validate whether the driver supports the current PCI device + (e.g. matching vendor/device IDs). +- ``init()``: Set up specific control registers or ring-buffer data structures. +- ``remove()``: Clean up structures before destruction. +- ``memcpy_start()``: Initiate DMA copies via the device. +- ``memcpy_wait()``: Poll and wait until previously initiated copies complete + safely. +- ``send_msi()``: Provoke the device to signal an MSI interrupt. + +Tests are required to set up a dedicated DMA region for the driver to use +within ``struct vfio_pci_driver`` to serve as the driver's main memory +allocation region (e.g. used for in-memory descriptor rings and hardware +buffers). Tests must provision and map this DMA region before completing driver +initialization. + +For a complete example of interacting with the driver framework from a test +perspective, refer to ``tools/testing/selftests/vfio/vfio_pci_driver_test.c``. + +When contributing a new driver implementation to the selftests library, the +primary acceptance criteria is that ``vfio_pci_driver_test`` passes using the +new driver and target device. + +Integration with KVM Selftests +------------------------------ + +libvfio is not strictly limited to VFIO selftests, it is also leveraged within +KVM selftests (``tools/testing/selftests/kvm/``). KVM selftests build their own +copy of libvfio and link against it, enabling libvfio to be used in KVM +selftests. + +This integration enables testing the interaction between KVM and VFIO and +IOMMUFD. For example, testing the end-to-end delivery of device interrupts into +running vCPUs. + +Contribution Guidelines +======================= + +- Prefer the shortlog prefix ``vfio: selftests: ...`` for commits that modify + files in ``tools/testing/selftests/vfio/``. +- Follow the Coding Style, Comments, Changelogs, and Function References + guidelines from the + :doc:`KVM x86 maintainers handbook </process/maintainer-kvm-x86>`.
diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst index 8eb9a5d..7f95c31 100644 --- a/Documentation/networking/filter.rst +++ b/Documentation/networking/filter.rst
@@ -183,12 +183,9 @@ BPF engine and instruction set ------------------------------ -Under tools/bpf/ there's a small helper tool called bpf_asm which can -be used to write low-level filters for example scenarios mentioned in the -previous section. Asm-like syntax mentioned here has been implemented in -bpf_asm and will be used for further explanations (instead of dealing with -less readable opcodes directly, principles are the same). The syntax is -closely modelled after Steven McCanne's and Van Jacobson's BPF paper. +The examples below use an assembly notation instead of less readable +opcodes. The notation is closely modelled after Steven McCanne's and +Van Jacobson's BPF paper. The BPF architecture consists of the following basic elements: @@ -201,8 +198,8 @@ store", addressable from 0 to 15 ======= ==================================================== -A program, that is translated by bpf_asm into "opcodes" is an array that -consists of the following elements (as already mentioned):: +A BPF program is an array of instructions with the following elements +(as already mentioned):: op:16, jt:8, jf:8, k:32 @@ -213,9 +210,8 @@ ways depending on the given instruction in op. The instruction set consists of load, store, branch, alu, miscellaneous -and return instructions that are also represented in bpf_asm syntax. This -table lists all bpf_asm instructions available resp. what their underlying -opcodes as defined in linux/filter.h stand for: +and return instructions. This table lists the assembly notation and what +the corresponding opcodes defined in linux/filter.h do: =========== =================== ===================== Instruction Addressing mode Description @@ -374,141 +370,14 @@ ret #-1 drop: ret #0 -The above example code can be placed into a file (here called "foo"), and -then be passed to the bpf_asm tool for generating opcodes, output that xt_bpf -and cls_bpf understands and can directly be loaded with. Example with above -ARP code:: +The ARP filter shown above corresponds to the following sock_filter +entries:: - $ ./bpf_asm foo - 4,40 0 0 12,21 0 1 2054,6 0 0 4294967295,6 0 0 0, - -In copy and paste C-like output:: - - $ ./bpf_asm -c foo { 0x28, 0, 0, 0x0000000c }, { 0x15, 0, 1, 0x00000806 }, { 0x06, 0, 0, 0xffffffff }, { 0x06, 0, 0, 0000000000 }, -In particular, as usage with xt_bpf or cls_bpf can result in more complex BPF -filters that might not be obvious at first, it's good to test filters before -attaching to a live system. For that purpose, there's a small tool called -bpf_dbg under tools/bpf/ in the kernel source directory. This debugger allows -for testing BPF filters against given pcap files, single stepping through the -BPF code on the pcap's packets and to do BPF machine register dumps. - -Starting bpf_dbg is trivial and just requires issuing:: - - # ./bpf_dbg - -In case input and output do not equal stdin/stdout, bpf_dbg takes an -alternative stdin source as a first argument, and an alternative stdout -sink as a second one, e.g. `./bpf_dbg test_in.txt test_out.txt`. - -Other than that, a particular libreadline configuration can be set via -file "~/.bpf_dbg_init" and the command history is stored in the file -"~/.bpf_dbg_history". - -Interaction in bpf_dbg happens through a shell that also has auto-completion -support (follow-up example commands starting with '>' denote bpf_dbg shell). -The usual workflow would be to ... - -* load bpf 6,40 0 0 12,21 0 3 2048,48 0 0 23,21 0 1 1,6 0 0 65535,6 0 0 0 - Loads a BPF filter from standard output of bpf_asm, or transformed via - e.g. ``tcpdump -iem1 -ddd port 22 | tr '\n' ','``. Note that for JIT - debugging (next section), this command creates a temporary socket and - loads the BPF code into the kernel. Thus, this will also be useful for - JIT developers. - -* load pcap foo.pcap - - Loads standard tcpdump pcap file. - -* run [<n>] - -bpf passes:1 fails:9 - Runs through all packets from a pcap to account how many passes and fails - the filter will generate. A limit of packets to traverse can be given. - -* disassemble:: - - l0: ldh [12] - l1: jeq #0x800, l2, l5 - l2: ldb [23] - l3: jeq #0x1, l4, l5 - l4: ret #0xffff - l5: ret #0 - - Prints out BPF code disassembly. - -* dump:: - - /* { op, jt, jf, k }, */ - { 0x28, 0, 0, 0x0000000c }, - { 0x15, 0, 3, 0x00000800 }, - { 0x30, 0, 0, 0x00000017 }, - { 0x15, 0, 1, 0x00000001 }, - { 0x06, 0, 0, 0x0000ffff }, - { 0x06, 0, 0, 0000000000 }, - - Prints out C-style BPF code dump. - -* breakpoint 0:: - - breakpoint at: l0: ldh [12] - -* breakpoint 1:: - - breakpoint at: l1: jeq #0x800, l2, l5 - - ... - - Sets breakpoints at particular BPF instructions. Issuing a `run` command - will walk through the pcap file continuing from the current packet and - break when a breakpoint is being hit (another `run` will continue from - the currently active breakpoint executing next instructions): - - * run:: - - -- register dump -- - pc: [0] <-- program counter - code: [40] jt[0] jf[0] k[12] <-- plain BPF code of current instruction - curr: l0: ldh [12] <-- disassembly of current instruction - A: [00000000][0] <-- content of A (hex, decimal) - X: [00000000][0] <-- content of X (hex, decimal) - M[0,15]: [00000000][0] <-- folded content of M (hex, decimal) - -- packet dump -- <-- Current packet from pcap (hex) - len: 42 - 0: 00 19 cb 55 55 a4 00 14 a4 43 78 69 08 06 00 01 - 16: 08 00 06 04 00 01 00 14 a4 43 78 69 0a 3b 01 26 - 32: 00 00 00 00 00 00 0a 3b 01 01 - (breakpoint) - > - - * breakpoint:: - - breakpoints: 0 1 - - Prints currently set breakpoints. - -* step [-<n>, +<n>] - - Performs single stepping through the BPF program from the current pc - offset. Thus, on each step invocation, above register dump is issued. - This can go forwards and backwards in time, a plain `step` will break - on the next BPF instruction, thus +1. (No `run` needs to be issued here.) - -* select <n> - - Selects a given packet from the pcap file to continue from. Thus, on - the next `run` or `step`, the BPF program is being evaluated against - the user pre-selected packet. Numbering starts just as in Wireshark - with index 1. - -* quit - - Exits bpf_dbg. - JIT compiler ------------ @@ -540,83 +409,6 @@ is discouraged and introspection through bpftool (under tools/bpf/bpftool/) is the generally recommended approach instead. -In the kernel source tree under tools/bpf/, there's bpf_jit_disasm for -generating disassembly out of the kernel log's hexdump:: - - # ./bpf_jit_disasm - 70 bytes emitted from JIT compiler (pass:3, flen:6) - ffffffffa0069c8f + <x>: - 0: push %rbp - 1: mov %rsp,%rbp - 4: sub $0x60,%rsp - 8: mov %rbx,-0x8(%rbp) - c: mov 0x68(%rdi),%r9d - 10: sub 0x6c(%rdi),%r9d - 14: mov 0xd8(%rdi),%r8 - 1b: mov $0xc,%esi - 20: callq 0xffffffffe0ff9442 - 25: cmp $0x800,%eax - 2a: jne 0x0000000000000042 - 2c: mov $0x17,%esi - 31: callq 0xffffffffe0ff945e - 36: cmp $0x1,%eax - 39: jne 0x0000000000000042 - 3b: mov $0xffff,%eax - 40: jmp 0x0000000000000044 - 42: xor %eax,%eax - 44: leaveq - 45: retq - - Issuing option `-o` will "annotate" opcodes to resulting assembler - instructions, which can be very useful for JIT developers: - - # ./bpf_jit_disasm -o - 70 bytes emitted from JIT compiler (pass:3, flen:6) - ffffffffa0069c8f + <x>: - 0: push %rbp - 55 - 1: mov %rsp,%rbp - 48 89 e5 - 4: sub $0x60,%rsp - 48 83 ec 60 - 8: mov %rbx,-0x8(%rbp) - 48 89 5d f8 - c: mov 0x68(%rdi),%r9d - 44 8b 4f 68 - 10: sub 0x6c(%rdi),%r9d - 44 2b 4f 6c - 14: mov 0xd8(%rdi),%r8 - 4c 8b 87 d8 00 00 00 - 1b: mov $0xc,%esi - be 0c 00 00 00 - 20: callq 0xffffffffe0ff9442 - e8 1d 94 ff e0 - 25: cmp $0x800,%eax - 3d 00 08 00 00 - 2a: jne 0x0000000000000042 - 75 16 - 2c: mov $0x17,%esi - be 17 00 00 00 - 31: callq 0xffffffffe0ff945e - e8 28 94 ff e0 - 36: cmp $0x1,%eax - 83 f8 01 - 39: jne 0x0000000000000042 - 75 07 - 3b: mov $0xffff,%eax - b8 ff ff 00 00 - 40: jmp 0x0000000000000044 - eb 02 - 42: xor %eax,%eax - 31 c0 - 44: leaveq - c9 - 45: retq - c3 - -For BPF JIT developers, bpf_jit_disasm, bpf_asm and bpf_dbg provides a useful -toolchain for developing and testing the kernel's JIT compiler. - BPF kernel internals -------------------- Internally, for the kernel interpreter, a different instruction set @@ -656,16 +448,11 @@ Testing ------- -Next to the BPF toolchain, the kernel also ships a test module that contains -various test cases for classic and eBPF that can be executed against -the BPF interpreter and JIT compiler. It can be found in lib/test_bpf.c and -enabled via Kconfig:: - - CONFIG_TEST_BPF=m - -After the module has been built and installed, the test suite can be executed -via insmod or modprobe against 'test_bpf' module. Results of the test cases -including timings in nsec can be found in the kernel log (dmesg). +The BPF selftests under tools/testing/selftests/bpf ship instruction +tests in test_progs, see jit_insns, which runs the eBPF +instructions over a range of operand magnitudes and signs, register +combinations, memory offsets and jump offsets against the BPF interpreter +or the JIT compiler, depending on the net.core.bpf_jit_enable setting. Misc ----
diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst index 103328d..93c85b6 100644 --- a/Documentation/networking/net_cachelines/tcp_sock.rst +++ b/Documentation/networking/net_cachelines/tcp_sock.rst
@@ -152,6 +152,7 @@ unsigned_int keepalive_time unsigned_int keepalive_intvl int linger2 +u32 bpf_tcp_ops_flags read_mostly read_mostly bpf_tcp_ops_hdr_opt_len,bpf_skops_write_hdr_opt(tx);bpf_tcp_ops_parse_hdr,tcp_bpf_rtt,tcp_cleanup_rbuf,tcp_queue_rcv,tcp_ofo_queue,tcp_fastopen_add_skb(rx); u8 bpf_sock_ops_cb_flags u8:1 bpf_chg_cc_inprogress u16 timeout_rehash
diff --git a/Documentation/tools/rtla/common_options.txt b/Documentation/tools/rtla/common_options.txt index 38da1cf..6bf0f86 100644 --- a/Documentation/tools/rtla/common_options.txt +++ b/Documentation/tools/rtla/common_options.txt
@@ -26,12 +26,19 @@ **--filter** *<filter>* - Filter the previous **-e** *sys:event* event with *<filter>*. For further information about event filtering see https://www.kernel.org/doc/html/latest/trace/events.html#event-filtering. + Filter the previous **-e** *sys:event* event with *<filter>*. + + If the same event is specified more than once but with different filters, the last specified filter will be the only one applied. + + For further information about event filtering see https://www.kernel.org/doc/html/latest/trace/events.html#event-filtering. This option cannot be unset. **--trigger** *<trigger>* Enable a trace event trigger to the previous **-e** *sys:event*. + + If the same event is specified more than once but with different triggers, the last specified trigger will be the only one applied. + If the *hist:* trigger is activated, the output histogram will be automatically saved to a file named *system_event_hist.txt*. For example, the command:
diff --git a/Documentation/tools/rtla/rtla-osnoise-top.rst b/Documentation/tools/rtla/rtla-osnoise-top.rst index b91c02a..b948a81 100644 --- a/Documentation/tools/rtla/rtla-osnoise-top.rst +++ b/Documentation/tools/rtla/rtla-osnoise-top.rst
@@ -28,6 +28,10 @@ ======= .. include:: common_osnoise_options.txt +**--ipi** + + Track sources of IPIs. + .. include:: common_top_options.txt .. include:: common_options.txt
diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index 7261f25..9423722 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst
@@ -1483,6 +1483,10 @@ printed in hexadecimal format. By default, this option is off. + funcgraph-retaddr + When set, the return address will always be printed. + By default, this option is off. + sleep-time When running function graph tracer, to include the time a task schedules out in its function. @@ -2829,7 +2833,7 @@ 0) 2.861 us | } /* putname() */ The return value of each traced function can be displayed after -an equal sign "=". When encountering system call failures, it +an equal sign "ret=". When encountering system call failures, it can be very helpful to quickly locate the function that first returns an error code. @@ -2839,16 +2843,16 @@ Example with funcgraph-retval:: 1) | cgroup_migrate() { - 1) 0.651 us | cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */ + 1) 0.651 us | cgroup_migrate_add_task(); /* ret=0xffff93fcfd346c00 */ 1) | cgroup_migrate_execute() { 1) | cpu_cgroup_can_attach() { 1) | cgroup_taskset_first() { - 1) 0.732 us | cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */ - 1) 1.232 us | } /* cgroup_taskset_first = 0xffff93fc8fb20000 */ - 1) 0.380 us | sched_rt_can_attach(); /* = 0x0 */ - 1) 2.335 us | } /* cpu_cgroup_can_attach = -22 */ - 1) 4.369 us | } /* cgroup_migrate_execute = -22 */ - 1) 7.143 us | } /* cgroup_migrate = -22 */ + 1) 0.732 us | cgroup_taskset_next(); /* ret=0xffff93fc8fb20000 */ + 1) 1.232 us | } /* cgroup_taskset_first ret=0xffff93fc8fb20000 */ + 1) 0.380 us | sched_rt_can_attach(); /* ret=0x0 */ + 1) 2.335 us | } /* cpu_cgroup_can_attach ret=-22 */ + 1) 4.369 us | } /* cgroup_migrate_execute ret=-22 */ + 1) 7.143 us | } /* cgroup_migrate ret=-22 */ The above example shows that the function cpu_cgroup_can_attach returned the error code -22 firstly, then we can read the code @@ -2859,43 +2863,55 @@ it will be printed in signed decimal format, otherwise it will printed in hexadecimal format. +When CONFIG_DEBUG_INFO_BTF is enabled, the smart format also takes the +declared return type into account. Signed integer and signed enum return +values are printed in signed decimal format, unsigned enum return values +are printed in unsigned decimal format, and unsigned integer, pointer, and +aggregate return values are printed in hexadecimal format. Boolean return +values are printed as ``true`` or ``false``. If the BTF type is unavailable, +the legacy error-code detection is used. + - smart: echo nofuncgraph-retval-hex > trace_options - hexadecimal: echo funcgraph-retval-hex > trace_options Example with funcgraph-retval-hex:: 1) | cgroup_migrate() { - 1) 0.651 us | cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */ + 1) 0.651 us | cgroup_migrate_add_task(); /* ret=0xffff93fcfd346c00 */ 1) | cgroup_migrate_execute() { 1) | cpu_cgroup_can_attach() { 1) | cgroup_taskset_first() { - 1) 0.732 us | cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */ - 1) 1.232 us | } /* cgroup_taskset_first = 0xffff93fc8fb20000 */ - 1) 0.380 us | sched_rt_can_attach(); /* = 0x0 */ - 1) 2.335 us | } /* cpu_cgroup_can_attach = 0xffffffea */ - 1) 4.369 us | } /* cgroup_migrate_execute = 0xffffffea */ - 1) 7.143 us | } /* cgroup_migrate = 0xffffffea */ + 1) 0.732 us | cgroup_taskset_next(); /* ret=0xffff93fc8fb20000 */ + 1) 1.232 us | } /* cgroup_taskset_first ret=0xffff93fc8fb20000 */ + 1) 0.380 us | sched_rt_can_attach(); /* ret=0x0 */ + 1) 2.335 us | } /* cpu_cgroup_can_attach ret=0xffffffea */ + 1) 4.369 us | } /* cgroup_migrate_execute ret=0xffffffea */ + 1) 7.143 us | } /* cgroup_migrate ret=0xffffffea */ -At present, there are some limitations when using the funcgraph-retval -option, and these limitations will be eliminated in the future: +Note that there are some limitations when using the funcgraph-retval +option: -- Even if the function return type is void, a return value will still - be printed, and you can just ignore it. +- If CONFIG_DEBUG_INFO_BTF is disabled (n), a return value is printed even for + functions with a void return type. When CONFIG_DEBUG_INFO_BTF is enabled (y), + the return value is printed only for non-void functions. -- Even if return values are stored in multiple registers, only the - value contained in the first register will be recorded and printed. - To illustrate, in the x86 architecture, eax and edx are used to store - a 64-bit return value, with the lower 32 bits saved in eax and the - upper 32 bits saved in edx. However, only the value stored in eax - will be recorded and printed. +- If a return value occupies multiple registers, only the value in the first + register is recorded and printed. For example, on the x86 architecture, a + 64-bit return value is stored across eax (lower 32 bits) and edx (upper 32 bits), + but only the contents of eax are captured. If CONFIG_DEBUG_INFO_BTF is enabled, + the suffix "(trunc)" is appended to the printed value to indicate that the + output may be truncated because high-order register contents are omitted. -- In certain procedure call standards, such as arm64's AAPCS64, when a - type is smaller than a GPR, it is the responsibility of the consumer - to perform the narrowing, and the upper bits may contain UNKNOWN values. - Therefore, it is advisable to check the code for such cases. For instance, - when using a u8 in a 64-bit GPR, bits [63:8] may contain arbitrary values, - especially when larger types are truncated, whether explicitly or implicitly. - Here are some specific cases to illustrate this point: +- Under certain procedure-call standards (e.g., arm64's AAPCS64), when the return + type is smaller than a general-purpose register (GPR), the caller is responsible + for narrowing the value; the upper bits of the register may contain undefined data. + For instance, when a u8 is returned in 64-bit GPR, bits [63:8] can hold arbitrary + values, especially when larger types are truncated (explicitly or implicitly). It + is therefore advisable to inspect the code in such cases. If CONFIG_DEBUG_INFO_BTF + is enabled (y), the return value is automatically trimmed to the width of the return + type. + + The following examples illustrate the behavior: **Case One**: @@ -2914,7 +2930,9 @@ RET If you pass 0x123456789abcdef to this function and want to narrow it, - it may be recorded as 0x123456789abcdef instead of 0xef. + it may be recorded as 0x123456789abcdef instead of 0xef. When + CONFIG_DEBUG_INFO_BTF is enabled, the value will be correctly truncated + to 0xef based on the size constraints of the u8 type. **Case Two**: @@ -2939,7 +2957,9 @@ RET When passing 0x2_0000_0000 to it, the return value may be recorded as - 0x2_0000_0000 instead of 0. + 0x2_0000_0000 instead of 0. When CONFIG_DEBUG_INFO_BTF is enabled, the + value will be correctly truncated to 0 based on the size constraints of + the int type. You can put some comments on specific functions by using trace_printk() For example, if you want to put a comment inside
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc..e95c9cd 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst
@@ -117,7 +117,7 @@ x86 includes both i386 and x86_64. Type: - system, vm, or vcpu. + system, vm, vcpu or guest_memfd. Parameters: what parameters are accepted by the ioctl. @@ -918,7 +918,8 @@ - KVM_ARM_IRQ_TYPE_SPI: in-kernel GICv2/GICv3: SPI, irq_id between 32 and 1019 (incl.) (the vcpu_index field is ignored) - in-kernel GICv5: SPI, irq_id between 0 and 65535 (incl.) + in-kernel GICv5: SPI, irq_id between 0 and the configured + number of SPIs minus one (1023 maximum) - KVM_ARM_IRQ_TYPE_PPI: in-kernel GICv2/GICv3: PPI, irq_id between 16 and 31 (incl.) in-kernel GICv5: PPI, irq_id between 0 and 127 (incl.) @@ -1322,6 +1323,9 @@ can still emulate all Arm exceptions by manipulating individual registers using the KVM_SET_ONE_REG API. +For a protected VM, setting ext_dabt_pending returns -EPERM; see +Documentation/virt/kvm/arm/pkvm.rst. Injecting an SError is unaffected. + See KVM_GET_VCPU_EVENTS for the data structure. Calling this ioctl on a vCPU that hasn't been initialized will return @@ -1545,7 +1549,9 @@ KVM_MP_STATE_RUNNABLE the vcpu is currently running [x86,arm64,riscv,loongarch] KVM_MP_STATE_UNINITIALIZED the vcpu is an application processor (AP) - which has not yet received an INIT signal [x86] + which has not yet received an INIT signal + [x86], or a protected VM's vcpu that the + guest holds powered off [arm64] KVM_MP_STATE_INIT_RECEIVED the vcpu has received an INIT signal, and is now ready for a SIPI [x86] KVM_MP_STATE_HALTED the vcpu has executed a HLT instruction and @@ -1624,6 +1630,12 @@ The only states that are valid are KVM_MP_STATE_STOPPED and KVM_MP_STATE_RUNNABLE which reflect if the vcpu should be paused or not. +On arm64, once a protected VM's vcpu has run, it reads as +KVM_MP_STATE_UNINITIALIZED while the guest holds it powered off, with +CPU_OFF or by not yet bringing it online with CPU_ON, and +KVM_MP_STATE_RUNNABLE and KVM_MP_STATE_SUSPENDED return -EPERM on it: only +the guest can power it on. See Documentation/virt/kvm/arm/pkvm.rst. + On LoongArch, only the KVM_MP_STATE_RUNNABLE state is used to reflect whether the vcpu is runnable. @@ -2324,7 +2336,8 @@ ENOENT no such register EINVAL invalid register ID, or no such register or used with VMs in protected virtualization mode on s390 - EPERM (arm64) register access not allowed before vcpu finalization + EPERM (arm64) register access not allowed before vcpu + finalization, or after a protected VM's vcpu has run EBUSY (riscv) changing register value not allowed after the vcpu has run at least once ====== ============================================================ @@ -2949,7 +2962,8 @@ ENOENT no such register EINVAL invalid register ID, or no such register or used with VMs in protected virtualization mode on s390 - EPERM (arm64) register access not allowed before vcpu finalization + EPERM (arm64) register access not allowed before vcpu + finalization, or after a protected VM's vcpu has run ======== ============================================================ (These error codes are indicative only: do not rely on a specific error @@ -3484,6 +3498,7 @@ ====== ================================================================= EINVAL the target is unknown, or the combination of features is invalid. ENOENT a features bit specified is unknown. + EPERM the vcpu belongs to a protected VM and has already run. ====== ================================================================= This tells KVM what type of CPU to present to the guest, and what @@ -3512,6 +3527,10 @@ state. All calls to this function after the initial call must use the same target and same set of feature flags, otherwise EINVAL will be returned. +For a protected VM this ioctl returns EPERM once the vcpu has run, and the +features it accepts are restricted: KVM_ARM_VCPU_PSCI_0_2 is required. See +Documentation/virt/kvm/arm/pkvm.rst. + Possible features: - KVM_ARM_VCPU_POWER_OFF: Starts the CPU in a power-off state. @@ -3600,20 +3619,14 @@ :Parameters: struct kvm_vcpu_init (out) :Returns: 0 on success; -1 on error -Errors: - - ====== ========================================== - ENODEV no preferred target available for the host - ====== ========================================== - This queries KVM for preferred CPU target type which can be emulated by KVM on underlying host. -The ioctl returns struct kvm_vcpu_init instance containing information -about preferred CPU target type and recommended features for it. The -kvm_vcpu_init->features bitmap returned will have feature bits set if -the preferred target recommends setting these features, but this is -not mandatory. +The ioctl returns a struct kvm_vcpu_init instance containing the +preferred CPU target type. The kvm_vcpu_init->features bitmap is +returned empty: userspace selects the vCPU features itself, and their +availability is reported by the capabilities listed under +KVM_ARM_VCPU_INIT. The information returned by this ioctl can be used to prepare an instance of struct kvm_vcpu_init for KVM_ARM_VCPU_INIT ioctl which will result in @@ -3676,7 +3689,7 @@ :Capability: KVM_CAP_ARM_SET_DEVICE_ADDR :Architectures: arm64 :Type: vm ioctl -:Parameters: struct kvm_arm_device_address (in) +:Parameters: struct kvm_arm_device_addr (in) :Returns: 0 on success, -1 on error Errors: @@ -3790,6 +3803,9 @@ KVM_CAP_GUEST_DEBUG_HW_WPS capabilities which return a positive number indicating the number of supported registers. +On arm64, this ioctl returns -EPERM for a protected VM: debugging a +protected guest is not supported. See Documentation/virt/kvm/arm/pkvm.rst. + For ppc, the KVM_CAP_PPC_GUEST_DEBUG_SSTEP capability indicates whether the single-step debug event (KVM_GUESTDBG_SINGLESTEP) is supported. @@ -6381,11 +6397,24 @@ KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated on-demand. -When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes -userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE -state. At VM creation time, all memory is shared, i.e. the PRIVATE attribute -is '0' for all gfns. Userspace can control whether memory is shared/private by -toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed. +When mapping a gfn into the guest, guest faults are always serviced from +guest_memfd regardless of whether memory is shared or private. KVM determines +shared vs. private based on the state in guest_memfd, which is the sole +authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` to +find out more about the creation-time shared/private status. Userspace can +control whether memory is shared/private by toggling +KVM_MEMORY_ATTRIBUTE_PRIVATE via :ref:`KVM_SET_MEMORY_ATTRIBUTES2` as needed. + +userspace_addr is expected to be the mmap()-ed address corresponding to the +right offset within the guest_memfd. Any mismatch between userspace_addr and +guest_memfd is not validated and is a user error. userspace_addr is only used +for host-side guest accesses such as kvm_read_guest(). + +If in-place conversion is disabled, KVM selects shared vs. private based on the +gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state. At VM creation time, all memory is +shared, i.e. the PRIVATE attribute is '0' for all gfns. Userspace can control +whether memory is shared/private by toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via +KVM_SET_MEMORY_ATTRIBUTES as needed. S390: ^^^^^ @@ -6393,6 +6422,8 @@ Returns -EINVAL if the VM has the KVM_VM_S390_UCONTROL flag set. Returns -EINVAL if called on a protected VM. +.. _KVM_SET_MEMORY_ATTRIBUTES: + 4.141 KVM_SET_MEMORY_ATTRIBUTES ------------------------------- @@ -6429,6 +6460,10 @@ The "flags" field is reserved for future extensions and must be '0'. +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES2`. + +.. _KVM_CREATE_GUEST_MEMFD: + 4.142 KVM_CREATE_GUEST_MEMFD ---------------------------- @@ -6483,10 +6518,10 @@ page tables. Private memory cannot. ============================ ================================================ -When the KVM MMU performs a PFN lookup to service a guest fault and the backing -guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be -consumed from guest_memfd, regardless of whether it is a shared or a private -fault. +When the KVM MMU performs a PFN lookup to service a guest fault, the fault will +always be consumed from guest_memfd, regardless of whether it is a shared or a +private fault (unless in-place conversion is disabled and the backing +guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP flag set). See KVM_SET_USER_MEMORY_REGION2 for additional details. @@ -6494,7 +6529,7 @@ --------------------------- :Capability: KVM_CAP_PRE_FAULT_MEMORY -:Architectures: none +:Architectures: x86, s390, arm64 :Type: vcpu ioctl :Parameters: struct kvm_pre_fault_memory (in/out) :Returns: 0 if at least one page is processed, < 0 on error @@ -6502,12 +6537,15 @@ Errors: ========== =============================================================== + EAGAIN A race occurred before progress was made, but a retry may succeed. EINVAL The specified `gpa` and `size` were invalid (e.g. not page aligned, causes an overflow, or size is zero), or the VM is UCONTROL (s390). ENOENT The specified `gpa` is outside defined memslots. + ENOEXEC The vCPU has not been initialised (arm64). EINTR An unmasked signal is pending and no page was processed. EFAULT The parameter address was invalid. + EHWPOISON A poisoned host page was encountered. EOPNOTSUPP Mapping memory for a GPA is unsupported by the hypervisor, and/or for the current vCPU state/mode. EIO unexpected error conditions (also causes a WARN) @@ -6527,7 +6565,17 @@ KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory for the current vCPU state. KVM maps memory as if the vCPU generated a stage-2 read page fault, e.g. faults in memory as needed, but doesn't break -CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed. +CoW. On arm64, KVM marks newly created stage-2 PTEs as Accessed, as it +does for any stage-2 fault, but leaves the Accessed state of existing PTEs +unchanged. On x86, KVM does not mark any newly created stage-2 PTE as +Accessed, and for s390 it is not applicable. + +On arm64, a GPA is interpreted as an IPA, and never interpreted as the IPA +of a nested guest. Pre-faulting only populates canonical stage-2 page +tables. + +The feature is not supported on arm64 if the protected KVM (pKVM) feature +is enabled. In the case of confidential VM types where there is an initial set up of private guest memory before the guest is 'finalized'/measured, this ioctl @@ -6542,7 +6590,7 @@ remaining range. If `size` > 0 on return, the caller can just issue the ioctl again with the same `struct kvm_map_memory` argument. -Shadow page tables cannot support this ioctl because they +On x86, shadow page tables cannot support this ioctl because they are indexed by virtual address or nested guest physical address. Calling this ioctl when the guest is using shadow page tables (for example because it is running a nested guest with nested page tables) @@ -6663,6 +6711,83 @@ Userspace should use the defined constants from ``<linux/kvm.h>`` rather than hardcoding bit positions. +.. _KVM_SET_MEMORY_ATTRIBUTES2: + +4.146 KVM_SET_MEMORY_ATTRIBUTES2 +--------------------------------- + +:Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES +:Architectures: all +:Type: guest_memfd ioctl +:Parameters: struct kvm_memory_attributes2 (in/out) +:Returns: 0 on success, <0 on error + +Errors: + + ========== =============================================================== + EINVAL The specified `offset` or `size` was invalid (e.g. not + page aligned, causes an overflow, or size is zero). + EFAULT The parameter address was invalid. + ENOMEM Ran out of memory trying to track private/shared state + EAGAIN Pages in the range have outstanding references, see + ``error_offset`` below. + ========== =============================================================== + +KVM_SET_MEMORY_ATTRIBUTES2 is an extension to +KVM_SET_MEMORY_ATTRIBUTES that supports returning (writing) values to +userspace. The original (pre-extension) fields are shared with +KVM_SET_MEMORY_ATTRIBUTES identically. + +Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES. + +:: + + struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 error_offset; + __u64 reserved[11]; + }; + + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) + +The capability KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES enumerates the attributes +that can be set via KVM_SET_MEMORY_ATTRIBUTES2. + +Set attributes for a range of offsets within a guest_memfd to +KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed +memory range for guest use. Even if the guest_memfd was created with +GUEST_MEMFD_FLAG_MMAP, after a successful call to set +KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable +into host userspace and will only be mappable by the guest. + +To allow the range to be mappable into host userspace again, call +KVM_SET_MEMORY_ATTRIBUTES2 on the guest_memfd again with +KVM_MEMORY_ATTRIBUTE_PRIVATE unset. + +KVM does not directly manipulate the memory contents of pages during +attribute updates. However, the process of setting these attributes, +which includes operations such as unmapping pages from the host or +stage-2 page tables, may result in side effects on memory contents +that vary across different trusted firmware implementations. + +If this ioctl returns -EAGAIN, the offset of the page with unexpected +refcounts will be returned in ``error_offset``. This can occur if +there are transient refcounts on the pages, taken by other parts of +the kernel. + +Userspace is expected to figure out how to remove all known refcounts +on the shared pages, such as refcounts taken by get_user_pages(), and +try the ioctl again. A possible source of these long term refcounts is +if the guest_memfd memory was pinned in IOMMU page tables. + +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`. + .. _kvm_run: 5. The kvm_run structure @@ -9635,6 +9760,31 @@ is present and the kernel has enabled in-kernel emulation of the local APIC. On newer versions, ``KVM_GET_SUPPORTED_CPUID`` does report the bit as available. +Long mode segment limits +~~~~~~~~~~~~~~~~~~~~~~~~ + +CPU[EAX=0x80000008]:EBX[20] (EFER_LMSLE_MBZ) is a "defeature" bit: it is set +when the CPU does *not* support long mode segment limits, and so requires +EFER.LMSLE to be zero. KVM reports the bit via ``KVM_GET_SUPPORTED_CPUID`` if +and only if KVM refuses to set EFER.LMSLE, i.e. if the CPU doesn't support long +mode segment limits, or if nested SVM is unsupported. KVM therefore reports +the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is +enabled. + +KVM never reports the bit via ``KVM_GET_EMULATED_CPUID``, but userspace may set +it via ``KVM_SET_CPUID2`` even on a host where KVM doesn't report it. KVM +honors the guest's enumeration and rejects EFER.LMSLE=1 accordingly. That lets +userspace defeature a vCPU on a host that *does* support long mode segment +limits, so that the vCPU can later be migrated to a host that doesn't, e.g. so +that a vCPU created on AMD Rome can be migrated to Milan and later, which +dropped support for long mode segment limits. The opposite direction needs no +emulation, as a host that lacks long mode segment limits already enumerates the +defeature. + +Note, ``KVM_SET_MSRS`` is exempt from the check, as host-initiated MSR writes +skip guest CPUID checks so that userspace can set MSRs before it sets guest +CPUID. ``KVM_SET_SREGS`` and nested VMRUN are not exempt. + CPU topology ~~~~~~~~~~~~
diff --git a/Documentation/virt/kvm/arm/fw-pseudo-registers.rst b/Documentation/virt/kvm/arm/fw-pseudo-registers.rst index d78b53b..c07471e 100644 --- a/Documentation/virt/kvm/arm/fw-pseudo-registers.rst +++ b/Documentation/virt/kvm/arm/fw-pseudo-registers.rst
@@ -33,6 +33,8 @@ highest PSCI version implemented by KVM and compatible with v0.2) - Allows any PSCI version implemented by KVM and compatible with v0.2 to be set with SET_ONE_REG + - Returns -EINVAL for a version below v1.1 on a protected VM (see + Documentation/virt/kvm/arm/pkvm.rst) - Affects the whole VM (even if the register view is per-vcpu) * KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
diff --git a/Documentation/virt/kvm/arm/pkvm.rst b/Documentation/virt/kvm/arm/pkvm.rst index 514992a..b75bb8ad 100644 --- a/Documentation/virt/kvm/arm/pkvm.rst +++ b/Documentation/virt/kvm/arm/pkvm.rst
@@ -15,7 +15,9 @@ Booting a host kernel with '``kvm-arm.mode=protected``' enables "Protected KVM" (pKVM). During boot, pKVM installs a stage-2 identity map page-table for the host and uses it to isolate the hypervisor -running at EL2 from the rest of the host running at EL1/0. +running at EL2 from the rest of the host running at EL1/0. pKVM supports +GICv3 only: GICv2 is not supported in protected mode, and neither are +GICv5-based guests. pKVM permits creation of protected virtual machines (pVMs) by passing the ``KVM_VM_TYPE_ARM_PROTECTED`` machine type identifier to the @@ -31,7 +33,7 @@ Isolation mechanisms ==================== -pKVM relies on a number of mechanisms to isolate PVMs from the host: +pKVM relies on a number of mechanisms to isolate pVMs from the host: CPU memory isolation -------------------- @@ -50,8 +52,9 @@ to some user-visible differences when compared to non-protected VMs, largely due to the lack of MMU notifiers: -* Memslots cannot be moved or deleted once the pVM has started running. -* Read-only memslots and dirty logging are not supported. +* Memslots cannot be moved or deleted once the pVM has started running + (``-EPERM``). +* Read-only memslots and dirty logging are not supported (``-EPERM``). * With the exception of swap, file-backed pages cannot be mapped into a pVM. * Donated pages are accounted against ``RLIMIT_MLOCK`` and so the VMM @@ -67,12 +70,19 @@ then it will either return ``-EFAULT`` or forcefully reclaim the memory pages. Reclaimed memory is zeroed by the hypervisor and a subsequent attempt to access it in the pVM will return ``-EFAULT`` - from the ``VCPU_RUN`` ioctl(). + from the ``KVM_RUN`` ioctl(). CPU state isolation ------------------- -Status: **Unimplemented.** +Status: CPU register state of protected vCPUs is managed entirely at EL2. + +pKVM performs the complete context switch for protected vCPUs within the +hypervisor. The hypervisor sets a protected vCPU's initial state, not the +host, and only what each exit needs is synchronised back. + +The user-visible consequences are described under `API behaviour for +protected VMs`_. DMA isolation using an IOMMU ---------------------------- @@ -89,13 +99,153 @@ memory with Trustzone as part of a "confused deputy" attack. The PSCI proxy ensures that CPUs always have the stage-2 identity map -installed when they are executing in the host. +installed when they are executing in the host. This proxy is distinct +from the PSCI handling provided to protected guests, which is described +under `API behaviour for protected VMs`_. Protected VM firmware (pvmfw) ----------------------------- Status: **Unimplemented.** +API behaviour for protected VMs +=============================== + +Protected vCPU state is owned by EL2 (see `CPU state isolation`_). The VMM +configures a vCPU before its first ``KVM_RUN``; afterwards the state is +private to the guest and the ioctls that access it return ``-EPERM``. The +vCPU ioctls' errors follow one rule: ``-EPERM`` means the host asked for +state that the guest owns, ``-EINVAL`` means the request is not valid for a +protected VM, and a feature pKVM doesn't support for protected VMs is +rejected the way KVM rejects one that isn't implemented (``-ENXIO`` from +``KVM_ARM_VCPU_PVTIME_CTRL``, for example). The ioctls themselves are +described in Documentation/virt/kvm/api.rst. + +Boot +---- + +A protected VM boots from a single primary vCPU. Before the first +``KVM_RUN``, the VMM prepares the boot state: + +* Set ``KVM_MP_STATE_RUNNABLE`` on the primary vCPU and + ``KVM_MP_STATE_STOPPED`` on every other vCPU. EL2 allows only one + RUNNABLE primary per protected VM. A second RUNNABLE vCPU fails at its + first ``KVM_RUN``. +* Set the primary vCPU's boot state with ``KVM_SET_ONE_REG``: the kernel + entry address in ``PC`` and the DTB pointer in ``x0``. + +``PC`` and ``x0`` are the only registers EL2 takes from the host. Other +pre-run writes are accepted, but the guest starts from the state the +hypervisor resets it to. + +Secondary vCPUs are started by the guest itself through PSCI ``CPU_ON`` +(see `Power state`_), which supplies their entry point and context ID. +The VMM cannot choose where they boot. + +vCPU state +---------- + +* ``KVM_GET_ONE_REG`` and ``KVM_SET_ONE_REG`` return ``-EPERM`` once the + vCPU has run. Before that, they access the host-side copy from which + EL2 builds the guest's boot state (see `Boot`_). +* ``KVM_ARM_VCPU_INIT`` accepts only the vCPU features that a protected + guest supports and returns ``-EINVAL`` otherwise. + ``KVM_ARM_VCPU_PSCI_0_2`` is required, as EL2 implements PSCI 1.1 for + the guest (see `Power state`_). ``KVM_ARM_VCPU_EL1_32BIT`` is not + supported: protected guests run in AArch64 only and see no AArch32 + support in ``ID_AA64PFR0_EL1``. Once the vCPU has run, + ``KVM_ARM_VCPU_INIT`` returns ``-EPERM``, as re-initialising it would + reset the host-side copy alone. +* ``KVM_SET_VCPU_EVENTS`` returns ``-EPERM`` for external-abort injection + (``ext_dabt_pending``), and ``KVM_CAP_ARM_INJECT_EXT_DABT`` is not + advertised to a protected VM. SError injection is unaffected. +* ``KVM_SET_GUEST_DEBUG`` returns ``-EPERM``, and ``KVM_CAP_SET_GUEST_DEBUG`` + is not advertised to a protected VM (see `Debug`_). + +Power state +----------- + +EL2 implements PSCI 1.1 for a protected guest. The calls that move a +vCPU's power state, ``CPU_ON`` and ``CPU_OFF``, are handled at EL2, and +the host cannot change the outcome: for ``CPU_ON`` the host only +schedules the target, which EL2 has already reset to the entry point +the guest chose, and for ``CPU_OFF`` it only stops scheduling it. +``AFFINITY_INFO`` is handled by the host, which returns ``OFF`` once +it has stopped scheduling every vCPU the query covers, as for a +non-protected VM. A vCPU becomes a valid ``CPU_ON`` target at its first +``KVM_RUN``, and before that EL2 returns ``INVALID_PARAMETERS``. The +platform calls, ``CPU_SUSPEND``, ``SYSTEM_OFF``, ``SYSTEM_RESET`` and +``SYSTEM_RESET2``, are forwarded to the host and behave as for a +non-protected VM, with the ``SYSTEM_*`` calls exiting to the VMM as +``KVM_EXIT_SYSTEM_EVENT``; the ``SYSTEM_RESET2`` reset type and cookie +are in the guest's registers, which ``KVM_GET_ONE_REG`` rejects once +the vCPU has run. Any other function returns ``NOT_SUPPORTED``, and +``PSCI_FEATURES`` reports the same set. Because the host handles those +forwarded calls, ``KVM_SET_ONE_REG`` on ``KVM_REG_ARM_PSCI_VERSION`` +returns ``-EINVAL`` for a version below 1.1. + +Once a vCPU has run, its power state follows the guest's PSCI calls, not +the VMM's. A vCPU that the guest has powered off with ``CPU_OFF``, or has +not yet brought online with ``CPU_ON``, reads as +``KVM_MP_STATE_UNINITIALIZED``, and only an in-guest ``CPU_ON`` can power +it on: ``KVM_SET_MP_STATE`` returns ``-EPERM`` for ``KVM_MP_STATE_RUNNABLE`` +and ``KVM_MP_STATE_SUSPENDED`` on it, accepts ``KVM_MP_STATE_STOPPED`` and +leaves it as it is, and never takes ``KVM_MP_STATE_UNINITIALIZED`` from +the VMM. ``KVM_MP_STATE_STOPPED`` still stops a vCPU EL2 has powered on, +and ``KVM_MP_STATE_RUNNABLE`` resumes it, so the VMM can pause it. + +Other interface differences +--------------------------- + +* ``KVM_CHECK_EXTENSION`` reports only the arm64 capabilities that pKVM + supports for protected guests, and ``KVM_ENABLE_CAP`` accepts only + those. Query them on the VM file descriptor: the system file descriptor + has no VM to filter against. The capabilities the generic KVM code + answers itself (``KVM_CAP_SYNC_MMU`` or ``KVM_CAP_DEVICE_CTRL``, for + example) are reported as for any VM, and don't mean the feature works + for a protected one. The filter does not cover every interface either: + device-fd configuration (for example the VGIC after + ``KVM_CREATE_DEVICE``) and vCPU attributes are unfiltered, so a feature + the filter excludes is rejected by the attribute itself. + ``KVM_ARM_VCPU_PVTIME_CTRL`` returns ``-ENXIO``, for example, as it does + where steal time isn't implemented. +* The vGIC of a protected VM remains host-managed: device creation, + configuration and interrupt injection all work as they do for a + non-protected VM. +* ``KVM_ARM_SET_COUNTER_OFFSET`` and ``KVM_ARM_GET_REG_WRITABLE_MASKS`` + return ``-EINVAL``: their capabilities are not advertised to a protected + VM, whose counter offset and ID registers are set by EL2. A protected + guest sees the physical timebase. +* A protected guest's first access to each page of memory exits to the + host, since the hypervisor cannot tell memory from a device before the + page is mapped. For a store, the host sees the value of the register + the syndrome names, clamped to the access width, and nothing else from + the register file. For a load, the register the syndrome names takes + the value the host returns, as for any emulated access. +* A protected guest that uses a feature it was not given, or executes an + ``SMC``, takes an undefined instruction exception from the hypervisor; + the host is not involved. +* Apart from PSCI, the hypervisor handles a protected guest's SMCCC + calls itself and does not involve the host. A function it does not + implement returns ``NOT_SUPPORTED``, and the host's SMCCC filter does + not apply: the VM device attribute ioctls (``KVM_ARM_VM_SMCCC_CTRL``) + return ``-EINVAL`` for a protected VM. +* The hypervisor can decode a trapped guest access only from the CPU's + instruction syndrome, which is provided only for a load or store of a + single general-purpose register. An access without one (for example a + load/store pair or a SIMD/FP access) cannot be decoded. For a + non-protected VM it can exit to the VMM as ``KVM_EXIT_ARM_NISV``. For a + protected VM it cannot be emulated by the VMM, so the guest takes a + synchronous external abort instead. + +Debug +----- + +Hardware-assisted debugging is not available to protected guests. +``MDSCR_EL1``, ``OSLAR_EL1``, ``OSLSR_EL1``, ``OSDLR_EL1``, and breakpoint +and watchpoint 0 are RAZ/WI; any other debug register access takes an +undefined instruction exception. + Resources =========
diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index 70b9162..8a691c8 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst
@@ -12,12 +12,39 @@ will act as the VM interrupt controller, requiring emulated user-space devices to inject interrupts to the VGIC instead of directly to CPUs. -Creating a guest GICv5 device requires a GICv5 host. The current VGICv5 device -only supports PPI interrupts. These can either be injected from emulated -in-kernel devices (such as the Arch Timer, or PMU), or via the KVM_IRQ_LINE -ioctl. +Creating a guest GICv5 device requires a GICv5 host. The VGICv5 device supports +PPI, SPI, and LPI interrupts. The PPI and SPI interrupts can either be injected +from emulated in-kernel devices (such as the Arch Timer, or PMU), or via the +KVM_IRQ_LINE ioctl. LPIs are not externally injected, but are handled in +hardware via the LPI IST. Their pending state is driven directly by the guest. Groups: + KVM_DEV_ARM_VGIC_GRP_ADDR + Attributes: + + KVM_VGIC_V5_ADDR_TYPE_IRS (rw, 64-bit) + Base address in the guest physical address space of the GICv5 IRS + (Interrupt Routing Service) register mappings. Only valid for + KVM_DEV_TYPE_ARM_VGIC_V5. This address needs to be 64K aligned and the + region covers 128 KByte - the IRS has a CONFIG_FRAME and a SETLPI_FRAME, + each of which is 64 KBytes in size. + + Setting the address of the IRS in GPA space is mandatory before VGIC + resources are mapped, as the IRS is responsible for handling SPIs and + LPIs. Failure to set the IRS address before the first vCPU run results in + an error. + + KVM_DEV_ARM_VGIC_GRP_NR_IRQS + Attributes: + + A value describing the number of SPIs for this GIC instance. This is + GICv5-specific: unlike GICv2/v3, the value does not include SGIs or PPIs. + The value ranges from 32 to KVM's VGICv5 maximum of 1024 SPIs, in + increments of 32. If userspace does not set this attribute, KVM uses 32 + SPIs by default. + + kvm_device_attr.addr points to a __u32 value. + KVM_DEV_ARM_VGIC_GRP_CTRL Attributes: @@ -48,3 +75,239 @@ -EFAULT Invalid guest ram access -EBUSY One or more VCPUS are running ======= ======================================================== + + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS + Attributes: + + The attr field of kvm_device_attr encodes two values:: + + bits: | 63 .... 32 | 31 .... 16 | 15 .... 0 | + values: | mpidr | RES | instr | + + The mpidr field encodes the CPU ID based on the affinity information in the + architecture defined MPIDR, and the field is encoded as follows:: + + | 63 .... 56 | 55 .... 48 | 47 .... 40 | 39 .... 32 | + | Aff3 | Aff2 | Aff1 | Aff0 | + + The instr field encodes the system register to access based on the fields + defined in the A64 instruction set encoding for system register access + (RES means the bits are reserved for future use and should be zero):: + + | 15 ... 14 | 13 ... 11 | 10 ... 7 | 6 ... 3 | 2 ... 0 | + | Op 0 | Op1 | CRn | CRm | Op2 | + + All system regs accessed through this API are (rw, 64-bit) and + kvm_device_attr.addr points to a __u64 value. + + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the + CPU specified by the mpidr field. + + The available registers are: + + ======================= =================================================== + ICC_ICSR_EL1 + ICC_PPI_ENABLER0_EL1 + ICC_PPI_ENABLER1_EL1 + ICC_PPI_SACTIVER0_EL1 ICC_PPI_CACTIVER0_EL1 is not supported. Writes to + ICC_PPI_SACTIVER0_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SACTIVER1_EL1 ICC_PPI_CACTIVER1_EL1 is not supported. Writes to + ICC_PPI_SACTIVER1_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SPENDR0_EL1 ICC_PPI_CPENDR0_EL1 is not supported. Writes to + ICC_PPI_SPENDR0_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SPENDR1_EL1 ICC_PPI_CPENDR1_EL1 is not supported. Writes to + ICC_PPI_SPENDR1_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_PRIORITYR0_EL1 + ICC_PPI_PRIORITYR1_EL1 + ICC_PPI_PRIORITYR2_EL1 + ICC_PPI_PRIORITYR3_EL1 + ICC_PPI_PRIORITYR4_EL1 + ICC_PPI_PRIORITYR5_EL1 + ICC_PPI_PRIORITYR6_EL1 + ICC_PPI_PRIORITYR7_EL1 + ICC_PPI_PRIORITYR8_EL1 + ICC_PPI_PRIORITYR9_EL1 + ICC_PPI_PRIORITYR10_EL1 + ICC_PPI_PRIORITYR11_EL1 + ICC_PPI_PRIORITYR12_EL1 + ICC_PPI_PRIORITYR13_EL1 + ICC_PPI_PRIORITYR14_EL1 + ICC_PPI_PRIORITYR15_EL1 + ICC_APR_EL1 + ICC_CR0_EL1 + ICC_PCR_EL1 + ======================= =================================================== + + Errors: + + ======= ============================================================= + -ENXIO Getting or setting this register is not supported + -EBUSY VCPU is running, or write attempted after a VCPU has run + -EINVAL Invalid mpidr or register value supplied + ======= ============================================================= + + + KVM_DEV_ARM_VGIC_GRP_IRS_REGS + Attributes: + The attr field of kvm_device_attr encodes the offset of the IRS register, + relative to the IRS CONFIG_FRAME base address. This is the address that + was provided via KVM_VGIC_V5_ADDR_TYPE_IRS when creating VGICv5 in the + first place. + + kvm_device_attr.addr points to a __u64 value whatever the width + of the addressed register (32/64 bits). 64 bit registers can only + be accessed with full length. + + Writes to read-only registers are ignored by the kernel except for + IRS_IDR0 - IRS_IDR2 and IRS_IDR5 - IRS_IDR7, which are sanity checked to + ensure that they match a sane config. + + IRS_IDR3 and IRS_IDR4 are RAZ/WI, as nested virtualization is not + supported. + + For registers without dedicated userspace accessors, getting or setting a + register uses the same emulated MMIO handlers as guest reads/writes. + Dedicated userspace accessors may instead save or restore migration state + without triggering guest-visible side effects. For example, restoring + IRS_IST_BASER only restores the emulated register state; any host LPI IST + allocation based on the restored IRS_IST_CFGR and IRS_IST_BASER state + happens when KVM_DEV_ARM_VGIC_GRP_IST is restored. + + Once an LPI IST has been allocated, IRS_IST_CFGR is immutable. Userspace + may write back its current value, but setting a different value is + rejected with -EINVAL. + + Once an LPI IST has been allocated, IRS_IST_BASER is immutable. + Userspace may write back its current value, but setting a different value + is rejected with -EINVAL. + + Errors: + + ======= ================================================================= + -ENXIO Offset does not correspond to any supported register + -EFAULT Invalid user pointer for attr->addr + -EINVAL Offset is not 32-bit aligned for 32-bit MMIO registers, not + 64-bit aligned for 64-bit registers, or the supplied register + value is not compatible with the configured VGICv5 IRS state + -EBUSY VGIC is not initialized, one or more VCPUs are running, or a + write is attempted after a VCPU has run + ======= ================================================================= + + KVM_DEV_ARM_VGIC_GRP_IST + Attributes: + This interface is used to either save the state of the IRS's Interrupt + State Tables (ISTs), or to restore them. A get operation saves IST state, + and a set operation restores IST state. kvm_device_attr.attr is reserved + and must be zero. + + The VGIC must be initialized before using this interface. Restore must be + performed before the VM has run. For restore, userspace must have already + restored the IRS state needed to describe any guest LPI IST. + + Saving first asks the IRS to save and quiesce the VM so that interrupt + state has been written back to the ISTs. KVM checks that the VM remains + quiesced while copying out the SPI and LPI IST state. + + kvm_device_attr.addr points to a struct kvm_vgic_v5_ist:: + + struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; + }; + + The SPI and LPI IST buffers contain one little-endian 32-bit IST entry per + interrupt, in interrupt number order. Only the architected 32-bit ISTE + state is exposed to userspace; host IST layout and metadata are private to + KVM. + + The SPI IST buffer is required and its size must be: + + nr_spis * sizeof(__u32) + + where nr_spis is the value returned by KVM_DEV_ARM_VGIC_GRP_NR_IRQS for + the VGICv5 device. For VGICv5 this value is the number of SPIs, not the + total number of interrupts. + + The LPI IST buffer is required if the guest's IRS_IST_CFGR and + IRS_IST_BASER state describes a valid LPI IST, and its size must be: + + BIT(lpi_id_bits) * sizeof(__u32) + + where lpi_id_bits is the LPI ID width configured in IRS_IST_CFGR. If no + LPI IST is configured, lpi_ist_addr and lpi_ist_size must both be zero. + + Errors: + + =========== ============================================================ + -EBUSY One or more VCPUs are running, the VGIC is not initialized, + restore was requested after the VM has run, an LPI IST + already exists, or the save operation completed but the VM + did not remain quiesced + -EINVAL attr->addr is NULL, the userspace IST descriptor is missing + a required buffer, supplies a non-zero LPI buffer when no + LPI IST is configured, has a buffer size that does not match + the configured VGICv5 IST state, or an internal VM table + operation rejected the VM state + -EFAULT Invalid user pointer for attr->addr, spi_ist_addr, or + lpi_ist_addr + -ENXIO Required per-VM VGICv5/IST backing state is missing or + inconsistent + -ENOMEM Restoring IST state failed while allocating the host LPI IST + or tracking pending interrupts + -ETIMEDOUT An IRS save/VM operation timed out + =========== ============================================================ + +IRS Save Sequence: +------------------ + +The following operations are required when saving the virtual GICv5 IRS: + +a) Save the ISTs by issuing KVM_GET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_IST. +b) Save the IRS MMIO register state by issuing KVM_GET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IRS_REGS. + +These two steps may be performed in either order. Saving the ISTs writes all +IST state to userspace-provided buffers and does not update guest memory. + +IRS Restore Sequence: +--------------------- + +The following ordering must be followed when restoring the virtual GICv5 and +IRS: + +a) Create vCPUs. +b) Provide the IRS base address by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_ADDR +c) Restore the number of SPIs by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_NR_IRQS. +d) Initialise the GIC - this sets up the default state and creates the SPI + IST - by issuing KVM_SET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_CTRL with + KVM_DEV_ARM_VGIC_CTRL_INIT +e) Restore the IRS MMIO register state by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IRS_REGS. The IRS registers may be restored in any + order, but any IRS_IDR* state and the IRS_IST_CFGR/IRS_IST_BASER state + describing an LPI IST must be restored before restoring the ISTs. KVM uses + the restored IRS_IST_CFGR and IRS_IST_BASER state to allocate the LPI IST + during the following step. +f) Restore the ISTs by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IST. + +The number of SPIs must be restored before VGIC initialization because +initialization allocates the SPI state and fixes the SPI range exposed by the +IRS ID registers. + +The IRS's MMIO registers must be restored prior to restoring the ISTs as these +are used to convey the number of LPIs the guest has configured to KVM. + +The various ``*_STATUSR`` registers are observational state in the current KVM +implementation. Userspace may save them for validation or debugging purposes, +but they are not required as restore input and do not need to be replayed during +restore. + +Then vCPUs can be started.
diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index deb5c51..7e23584 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst
@@ -135,7 +135,7 @@ Request that the VCPU uses the specified hardware PMU when creating guest events for the purpose of PMU emulation. The PMU identifier can be read from the "type" file for the desired PMU instance under /sys/devices (or, equivalent, -/sys/bus/even_source). This attribute is particularly useful on heterogeneous +/sys/bus/event_source). This attribute is particularly useful on heterogeneous systems where there are at least two CPU PMUs on the system. The PMU that is set for one VCPU will be used by all the other VCPUs. It isn't possible to set a PMU if a PMU event filter is already present. @@ -231,7 +231,8 @@ Specifies the base address of the stolen time structure for this VCPU. The base address must be 64 byte aligned and exist within a valid guest memory region. See Documentation/virt/kvm/arm/pvtime.rst for more information -including the layout of the stolen time structure. +including the layout of the stolen time structure. Stolen time is not +supported for protected VMs (see Documentation/virt/kvm/arm/pkvm.rst). 4. GROUP: KVM_VCPU_TSC_CTRL ===========================
diff --git a/Documentation/virt/kvm/devices/vm.rst b/Documentation/virt/kvm/devices/vm.rst index a4d39fa..78e63eb 100644 --- a/Documentation/virt/kvm/devices/vm.rst +++ b/Documentation/virt/kvm/devices/vm.rst
@@ -280,6 +280,24 @@ :Parameters: none :Returns: 0 +4.5. ATTRIBUTE: KVM_S390_VM_CRYPTO_ENABLE_APIE (w/o) +------------------------------------------------------- + +Allows user space to enable AP interpretive execution. + +:Parameters: none +:Returns: 0 for success + -EOPNOTSUPP if AP instructions not available for guest + +4.6. ATTRIBUTE: KVM_S390_VM_CRYPTO_DISABLE_APIE (w/o) +------------------------------------------------------- + +Allows user space to disable AP interpretive execution. + +:Parameters: none +:Returns: 0 for success + -EOPNOTSUPP if AP instructions not available for guest + 5. GROUP: KVM_S390_VM_MIGRATION ===============================
diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index bd04a90..eeb0812 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst
@@ -170,7 +170,7 @@ :: - struct kvm_sev_launch_update { + struct kvm_sev_launch_update_data { __u64 uaddr; /* userspace address to be encrypted (must be 16-byte aligned) */ __u32 len; /* length of the data to be encrypted (must be 16-byte aligned) */ }; @@ -316,7 +316,7 @@ If len is zero on entry, the measurement blob length is written to len and uaddr is unused. -Parameters (in): struct kvm_sev_attestation +Parameters (in): struct kvm_sev_attestation_report Returns: 0 on success, -negative on error @@ -377,7 +377,7 @@ :: - struct kvm_sev_launch_send_update_data { + struct kvm_sev_send_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -447,7 +447,7 @@ :: - struct kvm_sev_launch_receive_update_data { + struct kvm_sev_receive_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -503,7 +503,8 @@ It is required that the GPA ranges initialized by this command have had the KVM_MEMORY_ATTRIBUTE_PRIVATE attribute set in advance. See the documentation -for KVM_SET_MEMORY_ATTRIBUTES for more details on this aspect. +for KVM_SET_MEMORY_ATTRIBUTES/KVM_SET_MEMORY_ATTRIBUTES2 for more details on +this aspect. Upon success, this command is not guaranteed to have processed the entire range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of @@ -511,9 +512,14 @@ remaining range that has yet to be processed. The caller should continue calling this command until those fields indicate the entire range has been processed, e.g. ``len`` is 0, ``gfn_start`` is equal to the last GFN in the -range plus 1, and ``uaddr`` is the last byte of the userspace-provided source -buffer address plus 1. In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, -``uaddr`` will be ignored completely. +range plus 1, and ``uaddr`` (if specified) is the last byte of the +userspace-provided source buffer address plus 1. + +In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, ``uaddr`` will be +ignored completely. For all other page types, ``uaddr`` is optional if in-place +conversion is enabled (i.e. when the data had been written directly to +guest_memfd while the page was in the shared state) and is required if in-place +conversion is disabled. Parameters (in): struct kvm_sev_snp_launch_update
diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index 521ecf9..e0b6c11 100644 --- a/Documentation/virt/kvm/x86/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst
@@ -109,7 +109,7 @@ Usage: a0: guest physical address where host copies -"struct kvm_clock_offset" structure. +"struct kvm_clock_pairing" structure. a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0) is supported (corresponding to the host's CLOCK_REALTIME clock).
diff --git a/Documentation/virt/kvm/x86/intel-tdx.rst b/Documentation/virt/kvm/x86/intel-tdx.rst index 6a222e9..d8d9409 100644 --- a/Documentation/virt/kvm/x86/intel-tdx.rst +++ b/Documentation/virt/kvm/x86/intel-tdx.rst
@@ -158,6 +158,10 @@ Initialize @nr_pages TDX guest private memory starting from @gpa with userspace provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned. +If guest_memfd in-place conversion is enabled, pass 0 for @source_addr +to represent "no source page". A source page is required if in-place +conversion is not enabled or not supported. + Note, before calling this sub command, memory attribute of the range [gpa, gpa + nr_pages] needs to be private. Userspace can use KVM_SET_MEMORY_ATTRIBUTES to set the attribute.
diff --git a/MAINTAINERS b/MAINTAINERS index 9388da2..5121b32 100644 --- a/MAINTAINERS +++ b/MAINTAINERS
@@ -1191,8 +1191,8 @@ AMD IOMMU (AMD-VI) M: Joerg Roedel (AMD) <joro@8bytes.org> +M: Vasant Hegde <vasant.hegde@amd.com> R: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> -R: Vasant Hegde <vasant.hegde@amd.com> L: iommu@lists.linux.dev S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git @@ -4895,6 +4895,7 @@ BPF JIT for RISC-V (32-bit) M: Luke Nelson <luke.r.nels@gmail.com> M: Xi Wang <xi.wang@gmail.com> +M: Pu Lehui <pulehui@huawei.com> L: bpf@vger.kernel.org S: Maintained F: arch/riscv/net/ @@ -4902,7 +4903,7 @@ BPF JIT for RISC-V (64-bit) M: Björn Töpel <bjorn@kernel.org> -R: Pu Lehui <pulehui@huawei.com> +M: Pu Lehui <pulehui@huawei.com> R: Puranjay Mohan <puranjay@kernel.org> L: bpf@vger.kernel.org S: Maintained @@ -4959,8 +4960,6 @@ M: Vadim Fedorenko <vadim.fedorenko@linux.dev> L: bpf@vger.kernel.org S: Maintained -F: crypto/bpf_crypto_skcipher.c -F: include/linux/bpf_crypto.h F: kernel/bpf/crypto.c BPF [DOCUMENTATION] (Related to Standardization) @@ -5005,7 +5004,6 @@ F: kernel/bpf/ F: kernel/trace/bpf_trace.c F: lib/buildid.c -F: lib/test_bpf.c F: net/bpf/ F: net/core/filter.c F: net/sched/act_bpf.c @@ -5190,6 +5188,18 @@ N: bcm283* N: raspberrypi +BROADCOM BCM2712 IOMMU DRIVER +M: Daniel Drake <dan@reactivated.net> +M: Florian Fainelli <florian.fainelli@broadcom.com> +M: Jim Quinlan <james.quinlan@broadcom.com> +R: Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com> +L: iommu@lists.linux.dev +L: linux-rpi-kernel@lists.infradead.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu* +F: drivers/iommu/bcm2712-iommu* +F: drivers/iommu/generic_pt/fmt/*bcm2712* + BROADCOM BCM281XX/BCM11XXX/BCM216XX ARM ARCHITECTURE M: Florian Fainelli <florian.fainelli@broadcom.com> M: Ray Jui <rjui@broadcom.com> @@ -7774,6 +7784,7 @@ F: include/linux/dma-map-ops.h F: include/linux/dma-mapping.h F: include/linux/swiotlb.h +F: include/trace/events/dma.h F: kernel/dma/ DMA MAPPING & SCATTERLIST API [RUST] @@ -14424,6 +14435,8 @@ F: arch/powerpc/include/uapi/asm/kvm* F: arch/powerpc/kernel/kvm* F: arch/powerpc/kvm/ +F: tools/testing/selftests/kvm/*/powerpc/ +F: tools/testing/selftests/kvm/powerpc/ KERNEL VIRTUAL MACHINE FOR RISC-V (KVM/riscv) M: Anup Patel <anup@brainfault.org> @@ -14448,6 +14461,7 @@ S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: Documentation/virt/kvm/s390* +F: arch/s390/include/arm64/* F: arch/s390/include/asm/gmap_helpers.h F: arch/s390/include/asm/kvm* F: arch/s390/include/uapi/asm/kvm* @@ -14460,6 +14474,22 @@ F: tools/testing/selftests/kvm/*/s390/ F: tools/testing/selftests/kvm/s390/ +KERNEL VIRTUAL MACHINE for arm-on-s390 (KVM/s390) +R: Steffen Eiden <seiden@linux.ibm.com> +R: Andreas Grapentin <gra@linux.ibm.com> +R: Marc Zyngier <maz@kernel.org> +L: kvm@vger.kernel.org +L: linux-s390@vger.kernel.org +S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git +F: arch/s390/include/arm64/ +F: arch/s390/include/asm/aef.h +F: arch/s390/include/asm/kvm_host_arm64* +F: arch/s390/include/asm/sae* +F: arch/s390/kernel/aef.c +F: arch/s390/kvm/arm64/ +F: arch/s390/tools/*arm64* + KERNEL VIRTUAL MACHINE FOR X86 (KVM/x86) M: Sean Christopherson <seanjc@google.com> M: Paolo Bonzini <pbonzini@redhat.com> @@ -15230,10 +15260,10 @@ LIVE PATCHING M: Josh Poimboeuf <jpoimboe@kernel.org> -M: Jiri Kosina <jikos@kernel.org> M: Miroslav Benes <mbenes@suse.cz> M: Petr Mladek <pmladek@suse.com> -R: Joe Lawrence <joe.lawrence@redhat.com> +M: Joe Lawrence <joe.lawrence@redhat.com> +M: Song Liu <song@kernel.org> L: live-patching@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git @@ -27984,6 +28014,22 @@ F: kernel/trace/trace_osnoise.c F: kernel/trace/trace_sched_wakeup.c +TRACING RING BUFFER +M: Steven Rostedt <rostedt@goodmis.org> +M: Vincent Donnefort <vdonnefort@google.com> +M: Masami Hiramatsu <mhiramat@kernel.org> +L: linux-kernel@vger.kernel.org +L: linux-trace-kernel@vger.kernel.org +S: Maintained +Q: https://patchwork.kernel.org/project/linux-trace-kernel/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git +F: Documentation/trace/ring-buffer* +F: Documentation/trace/remotes.rst +F: include/linux/ring-buffer*.h +F: kernel/trace/*ring_buffer*.c +F: kernel/trace/remote*.c +F: tools/testing/selftests/ring-buffer/ + TRADITIONAL CHINESE DOCUMENTATION M: Chen-Yu Yeh <chenyou910331@gmail.com> M: Weijie Yuan <wy@wyuan.org> @@ -28941,6 +28987,7 @@ M: David Matlack <dmatlack@google.com> L: kvm@vger.kernel.org S: Maintained +F: Documentation/driver-api/vfio-selftests.rst F: tools/testing/selftests/vfio/ VFIO VIRTIO PCI DRIVER
diff --git a/Makefile b/Makefile index 0319cd7..77caa3b 100644 --- a/Makefile +++ b/Makefile
@@ -1800,6 +1800,7 @@ CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ + vmlinux.BTF.inline \ vmlinux.thinlto-index builtin.order \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \
diff --git a/arch/arc/Makefile b/arch/arc/Makefile index 868805f..4ea705a 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile
@@ -86,7 +86,14 @@ core-$(CONFIG_ARC_PLAT_AXS10X) += arch/arc/plat-axs10x/ core-$(CONFIG_ARC_SOC_HSDK) += arch/arc/plat-hsdk/ -libs-y += arch/arc/lib/ $(LIBGCC) +libs-y += arch/arc/lib/ + +ifdef CONFIG_DEBUG_INFO_BTF +export LIBGCC := $(LIBGCC) +libs-y += arch/arc/lib/libgcc.a +else +libs-y += $(LIBGCC) +endif boot := arch/arc/boot
diff --git a/arch/arc/include/asm/cmpxchg.h b/arch/arc/include/asm/cmpxchg.h index b30b209..dd5214a 100644 --- a/arch/arc/include/asm/cmpxchg.h +++ b/arch/arc/include/asm/cmpxchg.h
@@ -62,6 +62,8 @@ _prev_; \ }) +#define arch_cmpxchg_local arch_cmpxchg_relaxed + #else #define arch_cmpxchg(ptr, old, new) \ @@ -83,6 +85,8 @@ _prev_; \ }) +#define arch_cmpxchg_local arch_cmpxchg + #endif /*
diff --git a/arch/arc/lib/Makefile b/arch/arc/lib/Makefile index 30158ae..2807ee2 100644 --- a/arch/arc/lib/Makefile +++ b/arch/arc/lib/Makefile
@@ -13,3 +13,11 @@ else lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs.o endif + +# libgcc can contain complex floating-point debug types unsupported by BTF. +# Strip only the debug information from a local copy of the selected archive. +always-$(CONFIG_DEBUG_INFO_BTF) += libgcc.a +OBJCOPYFLAGS_libgcc.a := --strip-debug + +$(obj)/libgcc.a: $(LIBGCC) FORCE + $(call if_changed,objcopy)
diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index 882cd70..a6cf55c 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c
@@ -125,7 +125,7 @@ static int __init xen_mm_init(void) return 0; /* we can work with the default swiotlb */ - rc = swiotlb_init_late(swiotlb_size_or_default(), + rc = swiotlb_init_late(swiotlb_default_pool_size(), xen_swiotlb_gfp(), NULL); if (rc < 0) return rc;
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 04e1a0d..ed28f94 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile
@@ -45,6 +45,9 @@ KBUILD_CFLAGS += $(call cc-disable-warning, psabi) KBUILD_AFLAGS += $(compat_vdso) +# Enable all code shared to s390 +KBUILD_CPPFLAGS += -DARM64_S390_COMMON=1 + ifeq ($(call rustc-min-version, 108500),y) KBUILD_RUSTFLAGS += --target=aarch64-unknown-none-softfloat else
diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h index beb42c6..dd2d153 100644 --- a/arch/arm64/include/asm/brk-imm.h +++ b/arch/arm64/include/asm/brk-imm.h
@@ -2,6 +2,7 @@ /* * Copyright (C) 2012 ARM Ltd. */ +/* Whole file is shared with s390 */ #ifndef __ASM_BRK_IMM_H #define __ASM_BRK_IMM_H
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index f816f5d7..a584f88 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h
@@ -3,11 +3,12 @@ * Copyright (C) 2013 - ARM Ltd * Author: Marc Zyngier <marc.zyngier@arm.com> */ +/* Whole file is shared with s390 */ #ifndef __ASM_ESR_H #define __ASM_ESR_H -#include <asm/memory.h> +#include <asm/brk-imm.h> #include <asm/sysreg.h> #define ESR_ELx_EC_UNKNOWN UL(0x00) @@ -435,7 +436,36 @@ #define ESR_ELx_IT_GCSPOPX 7 #ifndef __ASSEMBLER__ -#include <asm/types.h> +#include <linux/types.h> + +static __always_inline bool esr_trap_is_iabt(unsigned long esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; +} + +/* Always check for S1PTW *before* using this. */ +static __always_inline bool esr_dabt_is_write(unsigned long esr) +{ + return esr & ESR_ELx_WNR; +} + +static __always_inline bool esr_dabt_is_cm(unsigned long esr) +{ + return esr & ESR_ELx_CM; +} + +static __always_inline bool esr_abt_is_sea(unsigned long esr) +{ + switch (esr & ESR_ELx_FSC) { + case ESR_ELx_FSC_EXTABT: + case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): + case ESR_ELx_FSC_SECC: + case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): + return true; + default: + return false; + } +} static inline unsigned long esr_brk_comment(unsigned long esr) {
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 4bfbd82..678f0da 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h
@@ -3,14 +3,16 @@ * Copyright (C) 2012,2013 - ARM Ltd * Author: Marc Zyngier <marc.zyngier@arm.com> */ +/* Whole file is shared with s390 */ #ifndef __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__ +#include <linux/const.h> +#include <linux/bits.h> +#include <linux/types.h> #include <asm/esr.h> -#include <asm/memory.h> #include <asm/sysreg.h> -#include <asm/types.h> /* * Because I'm terribly lazy and that repainting the whole of the KVM
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e5b92ac..7d799cd 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h
@@ -62,6 +62,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs, __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_alloc_selftest, MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM), @@ -87,14 +88,20 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___tracing_write_event, __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaim, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaimable, + __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, + __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY), /* Hypercalls that are available only when pKVM has finalised. */ - __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, - __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, __KVM_HOST_SMCCC_FUNC___pkvm_host_donate_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_share_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_guest,
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index a3c1928..85289a4 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -44,6 +44,7 @@ enum exception_type { bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); void kvm_skip_instr32(struct kvm_vcpu *vcpu); +#if ARM64_S390_COMMON == 1 void kvm_inject_undefined(struct kvm_vcpu *vcpu); void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr); int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr); @@ -75,6 +76,8 @@ static inline int kvm_inject_serror(struct kvm_vcpu *vcpu) void kvm_vcpu_wfi(struct kvm_vcpu *vcpu); +#endif /* ARM64_S390_COMMON == 1 */ + void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2); int kvm_inject_nested_irq(struct kvm_vcpu *vcpu); @@ -133,12 +136,37 @@ static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr) static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) { - return (unsigned long *)&vcpu_gp_regs(vcpu)->pc; + return (unsigned long *)&vcpu->arch.ctxt.regs.pc; } static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) { - return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate; + return (unsigned long *)&vcpu->arch.ctxt.regs.pstate; +} + +static __always_inline unsigned long *vcpu_sp_el0(struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.ctxt.regs.sp; +} + +static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu) +{ + return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); +} + +static __always_inline __u128 *vcpu_vreg(struct kvm_vcpu *vcpu, int n) +{ + return &vcpu->arch.ctxt.fp_regs.vregs[n]; +} + +static __always_inline __u32 *vcpu_fpsr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.ctxt.fp_regs.fpsr; +} + +static __always_inline __u32 *vcpu_fpcr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.ctxt.fp_regs.fpcr; } static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) @@ -159,6 +187,7 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT; } +#if ARM64_S390_COMMON == 1 /* * vcpu_get_reg and vcpu_set_reg should always be passed a register number * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on @@ -167,16 +196,18 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu, u8 reg_num) { - return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num]; + return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)[reg_num]; } static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, unsigned long val) { if (reg_num != 31) - vcpu_gp_regs(vcpu)->regs[reg_num] = val; + vcpu_gp_regs(vcpu)[reg_num] = val; } +#endif /* ARM64_S390_COMMON == 1 */ + static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt) { switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) { @@ -379,6 +410,12 @@ static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu) return vcpu->arch.fault.disr_el1; } +#if ARM64_S390_COMMON == 1 +static __always_inline bool esr_abt_is_s1ptw(unsigned long esr) +{ + return esr & ESR_ELx_S1PTW; +} + static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu) { return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK; @@ -411,18 +448,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW); + return esr_abt_is_s1ptw(kvm_vcpu_get_esr(vcpu)); } /* Always check for S1PTW *before* using this. */ static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR; -} - -static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu) -{ - return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM); + return esr_dabt_is_write(kvm_vcpu_get_esr(vcpu)); } static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) @@ -443,12 +475,7 @@ static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW; -} - -static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu); + return esr_trap_is_iabt(kvm_vcpu_get_esr(vcpu)); } static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) @@ -456,6 +483,13 @@ static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC; } +#endif /* ARM64_S390_COMMON == 1 */ + +static __always_inline bool esr_abt_is_exec_fault(unsigned long esr) +{ + return esr_trap_is_iabt(esr) && !esr_abt_is_s1ptw(esr); +} + static inline bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) { @@ -468,26 +502,9 @@ bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu) return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu)); } -static inline -u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu) -{ - unsigned long esr = kvm_vcpu_get_esr(vcpu); - - BUG_ON(!esr_fsc_is_permission_fault(esr)); - return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL)); -} - static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) { - switch (kvm_vcpu_trap_get_fault(vcpu)) { - case ESR_ELx_FSC_EXTABT: - case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): - case ESR_ELx_FSC_SECC: - case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): - return true; - default: - return false; - } + return esr_abt_is_sea(kvm_vcpu_get_esr(vcpu)); } static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) @@ -496,9 +513,10 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) return ESR_ELx_SYS64_ISS_RT(esr); } -static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +#if ARM64_S390_COMMON == 1 +static inline bool esr_abt_is_write_fault(unsigned long esr) { - if (kvm_vcpu_abt_iss1tw(vcpu)) { + if (esr_abt_is_s1ptw(esr)) { /* * Only a permission fault on a S1PTW should be * considered as a write. Otherwise, page tables baked @@ -511,15 +529,22 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) * first), then a permission fault to allow the flags * to be set. */ - return kvm_vcpu_trap_is_permission_fault(vcpu); + return esr_fsc_is_permission_fault(esr); } - if (kvm_vcpu_trap_is_iabt(vcpu)) + if (esr_trap_is_iabt(esr)) return false; - return kvm_vcpu_dabt_iswrite(vcpu); + return esr_dabt_is_write(esr); } +static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +{ + return esr_abt_is_write_fault(kvm_vcpu_get_esr(vcpu)); +} + +#endif /* ARM64_S390_COMMON == 1 */ + static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) { return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; @@ -561,6 +586,7 @@ static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) return vcpu_read_sys_reg(vcpu, r) & bit; } +#if ARM64_S390_COMMON == 1 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, unsigned long data, unsigned int len) @@ -636,6 +662,8 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) vcpu_set_flag((v), e); \ } while (0) +#endif /* ARM64_S390_COMMON == 1 */ + /* * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE * format if E2H isn't set. @@ -737,6 +765,12 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) } } +static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) +{ + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); +} + +#if ARM64_S390_COMMON == 1 /* Reset a vcpu's core registers. */ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) { @@ -750,15 +784,19 @@ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) pstate = VCPU_RESET_PSTATE_EL1; /* Reset core registers */ - memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); - memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + memset(vcpu_gp_regs(vcpu), 0, sizeof(vcpu_gp_regs(vcpu))); + *vcpu_pc(vcpu) = 0; + *vcpu_sp_el0(vcpu) = 0; + kvm_reset_fpsimd(vcpu); vcpu->arch.ctxt.spsr_abt = 0; vcpu->arch.ctxt.spsr_und = 0; vcpu->arch.ctxt.spsr_irq = 0; vcpu->arch.ctxt.spsr_fiq = 0; - vcpu_gp_regs(vcpu)->pstate = pstate; + *vcpu_cpsr(vcpu) = pstate; } +#endif /* ARM64_S390_COMMON == 1 */ + /* PSCI reset handling for a vcpu. */ static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, struct vcpu_reset_state *reset_state)
diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h new file mode 100644 index 0000000..5f77a8a --- /dev/null +++ b/arch/arm64/include/asm/kvm_hcall.h
@@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * The host<->hyp hypercall interface. + * + * Copyright (C) 2026 Google LLC + * Author: Fuad Tabba <fuad.tabba@linux.dev> + */ + +#ifndef __ARM64_KVM_HCALL_H__ +#define __ARM64_KVM_HCALL_H__ + +#include <linux/args.h> +#include <linux/arm-smccc.h> +#include <linux/bug.h> +#include <linux/errno.h> +#include <linux/types.h> + +#include <asm/barrier.h> +#include <asm/kvm_asm.h> +#include <asm/spectre.h> +#include <asm/virt.h> + +typedef u16 pkvm_handle_t; + +enum pkvm_topup_id { + PKVM_TOPUP_HYP_ALLOC, + NR_PKVM_TOPUP_HYP_IDS, + PKVM_TOPUP_HYP_ALLOC_SELFTEST, +}; + +struct kvm; +struct kvm_s2_mmu; +struct kvm_vcpu; +struct vgic_v3_cpu_if; +struct vgic_v5_cpu_if; + +/* + * A host VA carried by a hypercall argument. At EL2 such a pointer must not + * be dereferenced until it is translated with kern_hyp_va_host(); sparse + * flags any use that skips the translation. The tag describes the EL2 view + * only: the host dereferences its own VAs freely. + */ +#if defined(__KVM_NVHE_HYPERVISOR__) && defined(__CHECKER__) +#define __kern __attribute__((noderef, address_space(__kern))) +#else +#define __kern +#endif + +/* + * Hypercall signatures are declared as (type, name) argument pairs. + * __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP() + * in <linux/syscalls.h>. The ladder is indexed by list entries, two per + * argument; __KVM_HCALL_MAP_N() takes that count explicitly. + */ +#define __KVM_HCALL_MAP2(m, t, a, ...) m(t, a) +#define __KVM_HCALL_MAP4(m, t, a, ...) m(t, a), __KVM_HCALL_MAP2(m, __VA_ARGS__) +#define __KVM_HCALL_MAP6(m, t, a, ...) m(t, a), __KVM_HCALL_MAP4(m, __VA_ARGS__) +#define __KVM_HCALL_MAP8(m, t, a, ...) m(t, a), __KVM_HCALL_MAP6(m, __VA_ARGS__) +#define __KVM_HCALL_MAP10(m, t, a, ...) m(t, a), __KVM_HCALL_MAP8(m, __VA_ARGS__) +#define __KVM_HCALL_MAP12(m, t, a, ...) m(t, a), __KVM_HCALL_MAP10(m, __VA_ARGS__) +#define __KVM_HCALL_MAP_N(n, m, ...) CONCATENATE(__KVM_HCALL_MAP, n)(m, __VA_ARGS__) +#define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__) + +#define __KVM_HCALL_DECL(t, a) t a +#define __KVM_HCALL_LONG(t, a) unsigned long a +#define __KVM_HCALL_CAST(t, a) (__force t) a +#define __KVM_HCALL_ARGS(t, a) a + +#ifndef __KVM_NVHE_HYPERVISOR__ +#define __kvm_call_hyp_nvhe(f, ...) \ + ({ \ + struct arm_smccc_res res; \ + \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ + ##__VA_ARGS__, &res); \ + if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ + res.a1 = -EOPNOTSUPP; \ + \ + res.a1; \ + }) + +/* + * Generate a typed stub for each declared hypercall. kvm_call_hyp_nvhe() + * resolves to the stub, so a call with the wrong argument count or types + * fails to compile instead of being silently truncated to an SMCCC function + * number and a pile of registers. The stub inlines to the same SMCCC call + * the untyped macro used to make. + */ +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ + static __always_inline \ + ret nvhe_hvc_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) \ + { \ + return (ret)__kvm_call_hyp_nvhe(name, \ + __KVM_HCALL_MAP(__KVM_HCALL_ARGS, __VA_ARGS__));\ + } + +#define DECLARE_KVM_HOST_HCALL0(ret, name) \ + static __always_inline ret nvhe_hvc_##name(void) \ + { \ + return (ret)__kvm_call_hyp_nvhe(name); \ + } + +#define kvm_call_hyp_nvhe(f, ...) nvhe_hvc_##f(__VA_ARGS__) + +/* + * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, + * where the eret to EL1 acts as a context synchronization event. + */ +#define kvm_call_hyp(f, ...) \ + do { \ + if (has_vhe()) { \ + f(__VA_ARGS__); \ + isb(); \ + } else { \ + kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ + } \ + } while (0) + +#define kvm_call_hyp_ret(f, ...) \ + ({ \ + typeof(f(__VA_ARGS__)) ret; \ + \ + if (has_vhe()) { \ + ret = f(__VA_ARGS__); \ + } else { \ + ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ + } \ + \ + ret; \ + }) +#else /* __KVM_NVHE_HYPERVISOR__ */ +#define kvm_call_hyp(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) + +/* + * At EL2 each declaration emits the canonical signature of the hypercall, + * which DEFINE_KVM_HOST_HCALL() in hyp-main.c checks the handler + * definition against. + */ +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ + typedef ret kvm_host_hcall_sig_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)); +#define DECLARE_KVM_HOST_HCALL0(ret, name) \ + typedef ret kvm_host_hcall_sig_##name(void); +#endif /* __KVM_NVHE_HYPERVISOR__ */ + +/* Hypercalls that are unavailable once pKVM has finalised. */ +DECLARE_KVM_HOST_HCALL(int, __pkvm_init, + phys_addr_t, phys, unsigned long, size, + unsigned long *, per_cpu_base, u32, hyp_va_bits) +DECLARE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, + phys_addr_t, phys, size_t, size, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, + enum arm64_hyp_spectre_vector, slot) +DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) +DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) +DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) + +DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) + +/* Hypercalls that are always available and common to [nh]VHE/pKVM. */ +DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc, + struct kvm_vcpu __kern *, vcpu) +DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run, + struct kvm_vcpu __kern *, vcpu) +DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, + struct kvm_s2_mmu __kern *, mmu) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) +DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, + struct kvm_s2_mmu __kern *, mmu) +DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, + u64, cntvoff) +DECLARE_KVM_HOST_HCALL(int, __tracing_load, + void __kern *, desc_hva, size_t, desc_size) +DECLARE_KVM_HOST_HCALL0(void, __tracing_unload) +DECLARE_KVM_HOST_HCALL(int, __tracing_enable, + bool, enable) +DECLARE_KVM_HOST_HCALL(int, __tracing_swap_reader, + unsigned int, cpu) +DECLARE_KVM_HOST_HCALL(void, __tracing_update_clock, + u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) +DECLARE_KVM_HOST_HCALL(int, __tracing_reset, + unsigned int, cpu) +DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event, + unsigned short, id, bool, enable) +DECLARE_KVM_HOST_HCALL(void, __tracing_write_event, + u64, id) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_resident, + struct vgic_v5_cpu_if __kern *, cpu_if); +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident, + struct vgic_v5_cpu_if __kern *, cpu_if); +DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_vdpend, + u32, intid, bool, pending, u16, vm); +DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, + enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, + enum pkvm_topup_id, id, unsigned long, target) +DECLARE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, + enum pkvm_topup_id, id) + +/* Hypercalls that are available only when pKVM has finalised. */ +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, + u64, pfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, + u64, pfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, + u64, pfn, u64, gfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, + u64, pfn, u64, gfn, u64, nr_pages, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, + u64, gfn, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, + u64, gfn) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) +DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, + struct kvm __kern *, host_kvm, void __kern *, pgd_hva) +DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) +DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, + phys_addr_t, phys) +DECLARE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, + pkvm_handle_t, handle, u64, gfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, + pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) +DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) +DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) +DECLARE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, + pkvm_handle_t, handle) + +#endif /* __ARM64_KVM_HCALL_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 406c57c..6dd8462 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h
@@ -14,6 +14,7 @@ #include <linux/arm-smccc.h> #include <linux/bitmap.h> #include <linux/types.h> +#include <linux/interval_tree.h> #include <linux/jump_label.h> #include <linux/kvm_types.h> #include <linux/maple_tree.h> @@ -27,9 +28,13 @@ #include <asm/fpsimd.h> #include <asm/kvm.h> #include <asm/kvm_asm.h> +#include <asm/kvm_hcall.h> #include <asm/vncr_mapping.h> #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM +#define __KVM_HAVE_VCPU_RUN_PID_CHANGE #define KVM_HALT_POLL_NS_DEFAULT 500000 @@ -39,6 +44,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS +#if ARM64_S390_COMMON == 1 #define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) @@ -55,6 +61,51 @@ #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9) #define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10) #define KVM_REQ_VGIC_PROCESS_UPDATE KVM_ARCH_REQ(11) +#define KVM_REQ_RELOAD_GICv5 KVM_ARCH_REQ(12) + +/* Bit indices for kvm_arch::flags */ +enum { + /* + * If we encounter a data abort without valid instruction syndrome + * information, report this to user space. User space can (and + * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is + * supported. + */ + KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, + /* Memory Tagging Extension enabled for the guest */ + KVM_ARCH_FLAG_MTE_ENABLED, + /* At least one vCPU has ran in the VM */ + KVM_ARCH_FLAG_HAS_RAN_ONCE, + /* The vCPU feature set for the VM is configured */ + KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, + /* PSCI SYSTEM_SUSPEND enabled for the guest */ + KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, + /* VM counter offset */ + KVM_ARCH_FLAG_VM_COUNTER_OFFSET, + /* Timer PPIs made immutable */ + KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, + /* Initial ID reg values loaded */ + KVM_ARCH_FLAG_ID_REGS_INITIALIZED, + /* Fine-Grained UNDEF initialised */ + KVM_ARCH_FLAG_FGU_INITIALIZED, + /* SVE exposed to guest */ + KVM_ARCH_FLAG_GUEST_HAS_SVE, + /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ + KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, + /* Unhandled SEAs are taken to userspace */ + KVM_ARCH_FLAG_EXIT_SEA, + /* No further ID register changes possible */ + KVM_ARCH_FLAG_ID_REGS_FINAL, +}; + +struct vcpu_reset_state { + unsigned long pc; + unsigned long r0; + bool be; + bool reset; +}; + +#endif /* ARM64_S390_COMMON == 1 */ #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -93,9 +144,22 @@ struct kvm_hyp_memcache { struct pkvm_mapping *mapping; /* only used from EL1 */ #define HYP_MEMCACHE_ACCOUNT_STAGE2 BIT(1) +#define HYP_MEMCACHE_ACCOUNT_KMEMCG BIT(2) unsigned long flags; }; +static inline void init_hyp_memcache(struct kvm_hyp_memcache *mc) +{ + memset(mc, 0, sizeof(*mc)); + mc->mapping = ZERO_SIZE_PTR; /* Prevent allocation, solely useful for stage2 memcache */ +} + +static inline void init_hyp_stage2_memcache(struct kvm_hyp_memcache *mc) +{ + memset(mc, 0, sizeof(*mc)); + mc->flags = HYP_MEMCACHE_ACCOUNT_STAGE2 | HYP_MEMCACHE_ACCOUNT_KMEMCG; +} + static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc, phys_addr_t *p, phys_addr_t (*to_pa)(void *virt)) @@ -152,6 +216,36 @@ struct kvm_vmid { atomic64_t id; }; +/* + * Record of a guest stage-2 mapping, storing canonical and nested IPA + * ranges. Both ranges have the same size. The lower bits of nested.start + * store the index of the nested mmu this mapping belongs to. + */ +struct kvm_guest_s2_mapping { + struct interval_tree_node canonical; + struct interval_tree_node nested; +}; + +struct kvm_vcpu_ops { + void (*vcpu_load)(struct kvm_vcpu *vcpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); +}; + +struct kvm_gfn_range; +struct kvm_s2_fault_desc; + +struct kvm_vm_s2_ops { + bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + int (*vm_flush_remote_tlbs)(struct kvm *kvm); + int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, + u64 nr_pages); + void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, + bool may_block); + int (*vm_mem_abort)(const struct kvm_s2_fault_desc *s2fd); +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -213,22 +307,24 @@ struct kvm_s2_mmu { u64 tlb_vttbr; u64 tlb_vtcr; + /* Guest s2 mapping records indexed in this MMU's IPA space. */ + struct rb_root_cached guest_s2_mappings; + /* * true when this represents a nested context where virtual * HCR_EL2.VM == 1 */ bool nested_stage2_enabled; -#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS - struct dentry *shadow_pt_debugfs_dentry; -#endif - /* * true when this MMU needs to be unmapped before being used for a new * purpose. */ bool pending_unmap; + /* Index in the S2 MMU array, only valid for a shadow S2 */ + u16 s2_mmu_idx; + /* * 0: Nobody is currently using this, check vttbr for validity * >0: Somebody is actively using this. @@ -253,13 +349,9 @@ struct kvm_smccc_features { unsigned long vendor_hyp_bmap_2; /* Function numbers 64-127 */ }; -typedef u16 pkvm_handle_t; - struct kvm_protected_vm { pkvm_handle_t handle; - struct kvm_hyp_memcache teardown_mc; struct kvm_hyp_memcache stage2_teardown_mc; - bool is_protected; bool is_created; /* @@ -308,8 +400,24 @@ enum fgt_group_id { __NR_FGT_GROUP_IDS__ }; +enum kvm_arm_vm_flavor { + VM_NVHE, + VM_VHE, + /* VMs running on a hyp that doesn't trust */ + MARKER(__VM_DISTRUSTING_HYP), + VM_PKVM, /* Normal guests on pKVM */ + MARKER(__VM_PROTECTED), + VM_PROTECTED_PKVM, /* Protected VM */ + VM_FLAVOR_MAX +}; + struct kvm_arch { struct kvm_s2_mmu mmu; + const struct kvm_vm_s2_ops *vm_s2_ops; + + enum kvm_arm_vm_flavor vm_flavor; + /* Mandated version of PSCI */ + u32 psci_version; /* * Fine-Grained UNDEF, mimicking the FGT layout defined by the @@ -328,47 +436,21 @@ struct kvm_arch { size_t nested_mmus_size; int nested_mmus_next; + /* + * Serializes guest s2 tracking trees access when the mmu_lock + * is only held for read. + */ + spinlock_t guest_s2_tracking_lock; + /* Interrupt controller */ struct vgic_dist vgic; /* Timers */ struct arch_timer_vm_data timer_data; - /* Mandated version of PSCI */ - u32 psci_version; - /* Protects VM-scoped configuration data */ struct mutex config_lock; - /* - * If we encounter a data abort without valid instruction syndrome - * information, report this to user space. User space can (and - * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is - * supported. - */ -#define KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER 0 - /* Memory Tagging Extension enabled for the guest */ -#define KVM_ARCH_FLAG_MTE_ENABLED 1 - /* At least one vCPU has ran in the VM */ -#define KVM_ARCH_FLAG_HAS_RAN_ONCE 2 - /* The vCPU feature set for the VM is configured */ -#define KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED 3 - /* PSCI SYSTEM_SUSPEND enabled for the guest */ -#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 4 - /* VM counter offset */ -#define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 5 - /* Timer PPIs made immutable */ -#define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 6 - /* Initial ID reg values loaded */ -#define KVM_ARCH_FLAG_ID_REGS_INITIALIZED 7 - /* Fine-Grained UNDEF initialised */ -#define KVM_ARCH_FLAG_FGU_INITIALIZED 8 - /* SVE exposed to guest */ -#define KVM_ARCH_FLAG_GUEST_HAS_SVE 9 - /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ -#define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10 - /* Unhandled SEAs are taken to userspace */ -#define KVM_ARCH_FLAG_EXIT_SEA 11 unsigned long flags; /* VM-wide vCPU feature set */ @@ -837,17 +919,11 @@ extern s64 kvm_nvhe_sym(hyp_physvirt_offset); extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS]; #define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map) -struct vcpu_reset_state { - unsigned long pc; - unsigned long r0; - bool be; - bool reset; -}; - struct vncr_tlb; struct kvm_vcpu_arch { struct kvm_cpu_context ctxt; + const struct kvm_vcpu_ops *vcpu_ops; /* * Guest floating point state @@ -928,8 +1004,8 @@ struct kvm_vcpu_arch { /* Cache some mmu pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - /* Pages to top-up the pKVM/EL2 guest pool */ - struct kvm_hyp_memcache pkvm_memcache; + /* Pages to top-up the pKVM/EL2 guest stage-2 pool */ + struct kvm_hyp_memcache stage2_mc; /* Virtual SError ESR to restore when HCR_EL2.VSE is set */ u64 vsesr_el2; @@ -953,6 +1029,7 @@ struct kvm_vcpu_arch { pid_t pid; }; +#if ARM64_S390_COMMON == 1 /* * Each 'flag' is composed of a comma-separated triplet: * @@ -1035,11 +1112,28 @@ struct kvm_vcpu_arch { set; \ }) +#define __vcpu_copy_flag(vt, vs, flagset, f, m) \ + do { \ + typeof(vs->arch.flagset) tmp, val; \ + \ + __build_check_flag(vs, flagset, f, m); \ + \ + val = READ_ONCE(vs->arch.flagset); \ + val &= (m); \ + __vcpu_flags_preempt_disable(); \ + tmp = READ_ONCE(vt->arch.flagset); \ + tmp &= ~(m); \ + tmp |= val; \ + WRITE_ONCE(vt->arch.flagset, tmp); \ + __vcpu_flags_preempt_enable(); \ + } while (0) + #define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__) #define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__) #define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__) #define vcpu_test_and_clear_flag(v, ...) \ __vcpu_test_and_clear_flag((v), __VA_ARGS__) +#define vcpu_copy_flag(vt, vs, ...) __vcpu_copy_flag((vt), (vs), __VA_ARGS__) /* KVM_ARM_VCPU_INIT completed */ #define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0)) @@ -1048,6 +1142,11 @@ struct kvm_vcpu_arch { /* pKVM VCPU setup completed */ #define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2)) +int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); +bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); + +#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) + /* Exception pending */ #define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0)) /* @@ -1057,6 +1156,8 @@ struct kvm_vcpu_arch { #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1)) /* Target EL/MODE (not a single flag, but let's abuse the macro) */ #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1)) +/* Cover both PENDING_EXCEPTION and EXCEPT_MASK for global operations */ +#define PC_UPDATE_REQ __vcpu_single_flag(iflags, GENMASK(3, 0)) /* Host-set: the hyp flushes the non-protected vCPU state in on entry */ #define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4)) @@ -1106,6 +1207,8 @@ struct kvm_vcpu_arch { /* KVM is currently emulating an L2 to L1 exception */ #define IN_NESTED_EXCEPTION __vcpu_single_flag(sflags, BIT(9)) +#endif /* ARM64_S390_COMMON == 1 */ + #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_zcr_elx(vcpu) \ @@ -1151,6 +1254,9 @@ struct kvm_vcpu_arch { #define vcpu_has_ptrauth(vcpu) false #endif +#define kvm_id_regs_final(kvm) \ + test_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &(kvm)->arch.flags) + #define vcpu_on_unsupported_cpu(vcpu) \ vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU) @@ -1160,7 +1266,7 @@ struct kvm_vcpu_arch { #define vcpu_clear_on_unsupported_cpu(vcpu) \ vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU) -#define vcpu_gp_regs(v) (&(v)->arch.ctxt.regs) +#define vcpu_gp_regs(v) ((v)->arch.ctxt.regs.regs) /* * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the @@ -1241,14 +1347,6 @@ struct kvm_vcpu_stat { u64 exits; }; -unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); -int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); - -unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); - int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events); @@ -1260,51 +1358,6 @@ void kvm_arm_resume_guest(struct kvm *kvm); #define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid)) -#ifndef __KVM_NVHE_HYPERVISOR__ -#define kvm_call_hyp_nvhe(f, ...) \ - ({ \ - struct arm_smccc_res res; \ - \ - arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ - ##__VA_ARGS__, &res); \ - if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ - res.a1 = -EOPNOTSUPP; \ - \ - res.a1; \ - }) - -/* - * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, - * where the eret to EL1 acts as a context synchronization event. - */ -#define kvm_call_hyp(f, ...) \ - do { \ - if (has_vhe()) { \ - f(__VA_ARGS__); \ - isb(); \ - } else { \ - kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ - } \ - } while(0) - -#define kvm_call_hyp_ret(f, ...) \ - ({ \ - typeof(f(__VA_ARGS__)) ret; \ - \ - if (has_vhe()) { \ - ret = f(__VA_ARGS__); \ - } else { \ - ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ - } \ - \ - ret; \ - }) -#else /* __KVM_NVHE_HYPERVISOR__ */ -#define kvm_call_hyp(f, ...) f(__VA_ARGS__) -#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) -#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) -#endif /* __KVM_NVHE_HYPERVISOR__ */ - int handle_exit(struct kvm_vcpu *vcpu, int exception_index); void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index); @@ -1327,6 +1380,15 @@ int __init populate_nv_trap_config(void); void kvm_calculate_traps(struct kvm_vcpu *vcpu); +#if ARM64_S390_COMMON == 1 +unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); +int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); +int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); + +unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); + /* MMIO helpers */ void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data); unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); @@ -1334,6 +1396,8 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); int kvm_handle_mmio_return(struct kvm_vcpu *vcpu); int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa); +#endif /* ARM64_S390_COMMON == 1 */ + /* * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, * arrived in guest context. For arm64, any event that arrives while a vCPU is @@ -1348,7 +1412,7 @@ long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu); gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu); void kvm_update_stolen_time(struct kvm_vcpu *vcpu); -bool kvm_arm_pvtime_supported(void); +bool kvm_arm_pvtime_supported(struct kvm *kvm); int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu, @@ -1506,14 +1570,30 @@ struct kvm *kvm_arch_alloc_vm(void); #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected) +#ifdef __KVM_NVHE_HYPERVISOR__ +#define kvm_vm_is_protected(kvm) \ + (is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM)) +/* + * Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the + * pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu, + * which is populated correctly and is gated on is_protected_kvm_enabled(). + */ +#define vcpu_is_protected(vcpu) \ + ({ \ + struct kvm *__kvm = READ_ONCE((vcpu)->kvm); \ + \ + (__kvm && kvm_vm_is_protected(__kvm)); \ + }) + +#else + +#define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED) #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) -int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); -bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); +#define kvm_vm_hyp_is_distrusting(kvm) ((kvm)->arch.vm_flavor >= __VM_DISTRUSTING_HYP) -#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) +#endif /* __KVM_NVHE_HYPERVISOR__ */ #define kvm_has_mte(kvm) \ (system_supports_mte() && \ @@ -1531,11 +1611,15 @@ static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) return test_bit(feature, ka->vcpu_features); } +#if ARM64_S390_COMMON == 1 + #define kvm_vcpu_has_feature(k, f) __vcpu_has_feature(&(k)->arch, (f)) #define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f)) #define kvm_vcpu_initialized(v) vcpu_get_flag(v, VCPU_INITIALIZED) +#endif /* ARM64_S390_COMMON == 1 */ + int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM extern phys_addr_t hyp_mem_base; @@ -1547,6 +1631,7 @@ static inline void kvm_hyp_reserve(void) { } void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu); bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu); +bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu); static inline u64 *__vm_id_reg(struct kvm_arch *ka, u32 reg) {
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 4974492..b0e42be 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -10,6 +10,7 @@ #include <linux/compiler.h> #include <linux/kvm_host.h> #include <asm/alternative.h> +#include <asm/spectre.h> #include <asm/sysreg.h> DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); @@ -88,6 +89,8 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if); int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu); /* GICv5 */ +void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if); +void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if); /* No hypercalls for the following */ @@ -95,6 +98,7 @@ void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if); +void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm); #ifdef __KVM_NVHE_HYPERVISOR__ void __timer_enable_traps(struct kvm_vcpu *vcpu); @@ -150,6 +154,9 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val); extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); +extern enum mitigation_state kvm_nvhe_sym(spectre_v2_state); +extern enum mitigation_state kvm_nvhe_sym(spectre_v4_state); +extern enum mitigation_state kvm_nvhe_sym(spectre_bhb_state); extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); extern unsigned long kvm_nvhe_sym(hyp_nr_cpus); extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr);
diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h index 5f6e678..3396deb 100644 --- a/arch/arm64/include/asm/kvm_hypevents.h +++ b/arch/arm64/include/asm/kvm_hypevents.h
@@ -58,4 +58,18 @@ HYP_EVENT(selftest, ), RE_PRINTK("id=%llu", __entry->id) ); + +/* + * trace_hyp_printk() has too many specificities to be declared with HYP_EVENT(). + * However, we can use a REMOTE_EVENT macro to automatically do the declaration + * for the kernel side. + */ +REMOTE_EVENT_CUSTOM_PRINTK(hyp_printk, + 0, /* id will be overwritten during hyp event init */ + RE_STRUCT( + re_field(u16, fmt_id) + re_field(u64, args[]) + ), + __hyp_trace_printk(evt, seq) +); #endif
diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h index de133b7..16e8d60 100644 --- a/arch/arm64/include/asm/kvm_hyptrace.h +++ b/arch/arm64/include/asm/kvm_hyptrace.h
@@ -23,4 +23,13 @@ extern struct remote_event __hyp_events_end[]; extern struct hyp_event_id __hyp_event_ids_start[]; extern struct hyp_event_id __hyp_event_ids_end[]; +#define HYP_STRING_FMT_MAX_SIZE 127 + +struct hyp_string_fmt { + const char fmt[HYP_STRING_FMT_MAX_SIZE]; + u8 nr_args; +}; + +extern const struct hyp_string_fmt __hyp_string_fmts_start[]; +extern const struct hyp_string_fmt __hyp_string_fmts_end[]; #endif
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 6eae7e7..32392ab 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -7,6 +7,8 @@ #ifndef __ARM64_KVM_MMU_H__ #define __ARM64_KVM_MMU_H__ +#include <linux/compiler.h> + #include <asm/page.h> #include <asm/memory.h> #include <asm/mmu.h> @@ -126,6 +128,9 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) * replace the instructions with `nop`s. */ #ifndef __KVM_VHE_HYPERVISOR__ + if (is_protected_kvm_enabled() && !is_ttbr1_addr(v)) + return v; + asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n" /* mask with va_mask */ "ror %0, %0, #1\n" /* rotate to the first tag bit */ "add %0, %0, #0\n" /* insert the low 12 bits of the tag */ @@ -140,6 +145,14 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) #define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v)))) +/* + * Translate a __kern-tagged host VA, dropping the tag: the only sanctioned + * unwrap. Translation only, no ownership or bounds validation; the result + * carries the pointee type stripped of the tag and of any cv-qualifiers. + */ +#define kern_hyp_va_host(v) \ + ((TYPEOF_UNQUAL(*(v)) *)__kern_hyp_va((unsigned long)(__force void *)(v))) + extern u32 __hyp_va_bits; /* @@ -392,12 +405,10 @@ static inline bool kvm_supports_cacheable_pfnmap(void) #ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS void kvm_s2_ptdump_create_debugfs(struct kvm *kvm); -void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu); -void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu); +void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx); #else static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {} -static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) {} -static inline void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) {} +static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) {} #endif /* CONFIG_PTDUMP_STAGE2_DEBUGFS */ #endif /* __ASSEMBLER__ */
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 586026e..0f899ad 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h
@@ -78,6 +78,11 @@ extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, const union tlbi_info *info, void (*)(struct kvm_s2_mmu *, const union tlbi_info *)); +extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, + gpa_t nested_ipa, size_t map_size, + struct kvm_guest_s2_mapping *mapping); +extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, + gpa_t nipa, size_t size); extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); @@ -166,6 +171,8 @@ extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans); extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); extern void kvm_nested_s2_wp(struct kvm *kvm); +extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, + size_t unmap_size, bool may_block); extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); extern void kvm_nested_s2_flush(struct kvm *kvm);
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 41a8687..1c9c57c 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -7,10 +7,18 @@ #ifndef __ARM64_KVM_PGTABLE_H__ #define __ARM64_KVM_PGTABLE_H__ +#include <linux/bitfield.h> #include <linux/bits.h> -#include <linux/kvm_host.h> +#include <linux/kvm_types.h> +#include <linux/rbtree_types.h> +#include <linux/rcupdate.h> #include <linux/types.h> +#include <asm/memory.h> +#include <asm/page.h> + +struct kvm_s2_mmu; + #define KVM_PGTABLE_FIRST_LEVEL -1 #define KVM_PGTABLE_LAST_LEVEL 3 @@ -532,7 +540,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); * kvm_get_vtcr() - Helper to construct VTCR_EL2 * @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register. * @mmfr1: Sanitized value of SYS_ID_AA64MMFR1_EL1 register. - * @phys_shfit: Value to set in VTCR_EL2.T0SZ. + * @phys_shift: IPA size, in bits. * * The VTCR value is common across all the physical CPUs on the system. * We use system wide sanitised values to fill in different fields, @@ -859,6 +867,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * @addr: Input address for the start of the walk. * @ptep: Pointer to storage for the retrieved PTE. * @level: Pointer to storage for the level of the retrieved PTE. + * @flags: Flags to control the page-table walk + * (see struct kvm_pgtable_visit_ctx). * * The offset of @addr within a page is ignored. * @@ -869,7 +879,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * Return: 0 on success, negative error code on failure. */ int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, s8 *level); + kvm_pte_t *ptep, s8 *level, + enum kvm_pgtable_walk_flags flags); /** * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index cad6056..e756277 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -7,9 +7,9 @@ #define __ARM64_KVM_PKVM_H__ #include <linux/arm_ffa.h> +#include <linux/kvm_host.h> #include <linux/memblock.h> #include <linux/scatterlist.h> -#include <asm/kvm_host.h> #include <asm/kvm_pgtable.h> /* Maximum number of VMs that can co-exist under pKVM. */ @@ -17,7 +17,8 @@ #define HYP_MEMBLOCK_REGIONS 128 -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); +void pkvm_selftests(void); +int pkvm_init_host_vm(struct kvm *kvm); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); @@ -42,18 +43,51 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) case KVM_CAP_ARM_VM_IPA_SIZE: case KVM_CAP_ARM_PTRAUTH_ADDRESS: case KVM_CAP_ARM_PTRAUTH_GENERIC: + case KVM_CAP_ONE_REG: + case KVM_CAP_MP_STATE: + case KVM_CAP_VCPU_EVENTS: + case KVM_CAP_VCPU_ATTRIBUTES: + case KVM_CAP_IMMEDIATE_EXIT: + case KVM_CAP_IOEVENTFD: + case KVM_CAP_IRQFD_RESAMPLE: + case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: + case KVM_CAP_ARM_INJECT_SERROR_ESR: return true; case KVM_CAP_ARM_MTE: - return false; case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + case KVM_CAP_PRE_FAULT_MEMORY: return false; default: - return !kvm || !kvm_vm_is_protected(kvm); + return !kvm || (kvm->arch.vm_flavor == VM_PKVM); } } /* + * The vCPU features a protected VM may use: checked by the host at + * KVM_ARM_VCPU_INIT, applied by EL2 when the hyp VM is created. + */ +static inline void kvm_pkvm_vcpu_allowed_features(struct kvm *kvm, + unsigned long *allowed) +{ + bitmap_zero(allowed, KVM_VCPU_MAX_FEATURES); + + set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) + set_bit(KVM_ARM_VCPU_PMU_V3, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) + set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) + set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) + set_bit(KVM_ARM_VCPU_SVE, allowed); +} + +/* * Check whether the KVM VM IOCTL is allowed in pKVM. * * Certain features are allowed only for non-protected VMs in pKVM, which is why @@ -204,6 +238,116 @@ struct pkvm_mapping { u64 __subtree_last; /* Internal member for interval tree */ }; +enum pkvm_hyp_req_type { + PKVM_HYP_NO_REQ = 0, + PKVM_HYP_REQ_HYP_ALLOC, + PKVM_HYP_REQ_HYP_ALLOC_SELFTEST, + __PKVM_HYP_REQ_TYPE_MAX, +}; + +#define PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX \ + (sizeof(struct arm_smccc_res) - offsetof(struct arm_smccc_res, a2) - 1) + +struct pkvm_hyp_req { + u8 type; + union { + struct { + u32 nr_pages; + } mem; + struct { + /* Helper for SMCCC encoding/decoding */ + u8 args[PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX]; + } args; + }; +}; + +static inline size_t pkvm_hyp_req_arg_size(u8 type) +{ + struct pkvm_hyp_req *req; + + switch (type) { + case PKVM_HYP_NO_REQ: + return 0; + case PKVM_HYP_REQ_HYP_ALLOC: + case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: + return sizeof(req->mem); + default: + WARN_ON(1); + } + + return 0; +} + +/* Encode the pending pkvm_hyp_req type into the SMCCC args */ +static inline void +pkvm_hyp_req_to_smccc(struct kvm_cpu_context *host_ctxt, struct pkvm_hyp_req *req) +{ + u8 *dst, type = req->type; + size_t size; + + if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) { + host_ctxt->regs.regs[2] = 0; + return; + } + + size = pkvm_hyp_req_arg_size(type); + if (WARN_ON(size > PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX)) + return; + + dst = (u8 *)&host_ctxt->regs.regs[2]; + *dst = type; + + memcpy(dst + 1, &req->args, size); +} + +/* Return true if a pkvm_hyp_req has been decoded from the SMCCC args */ +static inline bool smccc_to_pkvm_hyp_req(struct pkvm_hyp_req *req, struct arm_smccc_res *res) +{ + u8 *src = (u8 *)res + offsetof(struct arm_smccc_res, a2); + u8 type = *src; + + if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) + return false; + + req->type = type; + memcpy(&req->args, src + 1, pkvm_hyp_req_arg_size(type)); + + return true; +} + +int __pkvm_handle_smccc_req(struct arm_smccc_res *res); + +/** + * pkvm_call_hyp_req() - Issue an HVC that can return hypervisor requests + * @f: Hypervisor function symbol to call. + * @...: Arguments to pass to the hypercall. + * + * Re-issue an HVC and process any pending hypervisor request until completion + * or error. + * + * Only use this helper for HVCs whose hypervisor handlers format their return + * registers with pkvm_hyp_req_to_smccc(). + * + * Return: Result of the hypercall or a negative error if the hyp request + * handling failed. + */ +#define pkvm_call_hyp_req(f, ...) \ +({ \ + struct arm_smccc_res __res; \ + int __ret; \ + do { \ + __ret = -1; \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), ##__VA_ARGS__, &__res); \ + if (WARN_ON(__res.a0 != SMCCC_RET_SUCCESS)) \ + break; \ + __ret = __res.a1; \ + if (!__ret) \ + break; \ + __ret = __pkvm_handle_smccc_req(&__res); \ + } while (!__ret); \ + __ret; \ +}) + int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops); void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index bd291bd..ad0ad30 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h
@@ -26,6 +26,8 @@ #define GIC_PRIO_PSR_I_SET GICV3_PRIO_PSR_I_SET +#if ARM64_S390_COMMON == 1 + /* Additional SPSR bits not exposed in the UABI */ #define PSR_MODE_THREAD_BIT (1 << 0) #define PSR_IL_BIT (1 << 20) @@ -66,6 +68,8 @@ #define PSR_AA32_IT_MASK 0x0600fc00 /* If-Then execution state mask */ #define PSR_AA32_GE_MASK 0x000f0000 +#endif /* ARM64_S390_COMMON == 1 */ + /* AArch32 CPSR bits, as seen in AArch32 */ #define COMPAT_PSR_DIT_BIT 0x00200000 @@ -96,26 +100,48 @@ #define COMPAT_USER_SZ 296 /* Architecturally defined mapping between AArch32 and AArch64 registers */ +enum { + __compat_fp = 11, + __compat_sp = 13, + __compat_lr = 14, + __compat_sp_hyp = 15, + __compat_lr_irq = 16, + __compat_sp_irq = 17, + __compat_lr_svc = 18, + __compat_sp_svc = 19, + __compat_lr_abt = 20, + __compat_sp_abt = 21, + __compat_lr_und = 22, + __compat_sp_und = 23, + __compat_r8_fiq = 24, + __compat_r9_fiq = 25, + __compat_r10_fiq = 26, + __compat_r11_fiq = 27, + __compat_r12_fiq = 28, + __compat_sp_fiq = 29, + __compat_lr_fiq = 30, +}; + #define compat_usr(x) regs[(x)] -#define compat_fp regs[11] -#define compat_sp regs[13] -#define compat_lr regs[14] -#define compat_sp_hyp regs[15] -#define compat_lr_irq regs[16] -#define compat_sp_irq regs[17] -#define compat_lr_svc regs[18] -#define compat_sp_svc regs[19] -#define compat_lr_abt regs[20] -#define compat_sp_abt regs[21] -#define compat_lr_und regs[22] -#define compat_sp_und regs[23] -#define compat_r8_fiq regs[24] -#define compat_r9_fiq regs[25] -#define compat_r10_fiq regs[26] -#define compat_r11_fiq regs[27] -#define compat_r12_fiq regs[28] -#define compat_sp_fiq regs[29] -#define compat_lr_fiq regs[30] +#define compat_fp regs[__compat_fp] +#define compat_sp regs[__compat_sp] +#define compat_lr regs[__compat_lr] +#define compat_sp_hyp regs[__compat_sp_hyp] +#define compat_lr_irq regs[__compat_lr_irq] +#define compat_sp_irq regs[__compat_sp_irq] +#define compat_lr_svc regs[__compat_lr_svc] +#define compat_sp_svc regs[__compat_sp_svc] +#define compat_lr_abt regs[__compat_lr_abt] +#define compat_sp_abt regs[__compat_sp_abt] +#define compat_lr_und regs[__compat_lr_und] +#define compat_sp_und regs[__compat_sp_und] +#define compat_r8_fiq regs[__compat_r8_fiq] +#define compat_r9_fiq regs[__compat_r9_fiq] +#define compat_r10_fiq regs[__compat_r10_fiq] +#define compat_r11_fiq regs[__compat_r11_fiq] +#define compat_r12_fiq regs[__compat_r12_fiq] +#define compat_sp_fiq regs[__compat_sp_fiq] +#define compat_lr_fiq regs[__compat_lr_fiq] static inline unsigned long compat_psr_to_pstate(const unsigned long psr) {
diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/asm/stacktrace/nvhe.h index 171f9ed..a631a57 100644 --- a/arch/arm64/include/asm/stacktrace/nvhe.h +++ b/arch/arm64/include/asm/stacktrace/nvhe.h
@@ -37,6 +37,14 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state, state->pc = pc; } +DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); +DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); + +#ifdef CONFIG_PKVM_STACKTRACE +DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], + pkvm_stacktrace); +#endif + #ifndef __KVM_NVHE_HYPERVISOR__ /* * Conventional (non-protected) nVHE HYP stack unwinder @@ -45,8 +53,6 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state, * (by the host in EL1). */ -DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); -DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); DECLARE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base); void kvm_nvhe_dump_backtrace(unsigned long hyp_offset);
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index cec3660..e3b60ee 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h
@@ -16,6 +16,8 @@ #include <asm/gpr-num.h> +#if ARM64_S390_COMMON == 1 + /* * ARMv8 ARM reserves the following encoding for system registers: * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", @@ -50,6 +52,8 @@ #define sys_reg_CRm(id) (((id) >> CRm_shift) & CRm_mask) #define sys_reg_Op2(id) (((id) >> Op2_shift) & Op2_mask) +#endif /* ARM64_S390_COMMON == 1 */ + #ifndef CONFIG_BROKEN_GAS_INST #ifdef __ASSEMBLER__ @@ -114,6 +118,8 @@ #define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31) #define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31) +#if ARM64_S390_COMMON == 1 + /* Data cache zero operations */ #define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2) #define SYS_DC_IGSW sys_insn(1, 0, 7, 6, 4) @@ -286,7 +292,6 @@ #define SYS_APGAKEYLO_EL1 sys_reg(3, 0, 2, 3, 0) #define SYS_APGAKEYHI_EL1 sys_reg(3, 0, 2, 3, 1) -#define SYS_SPSR_EL1 sys_reg(3, 0, 4, 0, 0) #define SYS_ELR_EL1 sys_reg(3, 0, 4, 0, 1) #define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0) @@ -509,7 +514,6 @@ #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) -#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0) #define SYS_SPSR_irq sys_reg(3, 4, 4, 3, 0) @@ -595,7 +599,6 @@ #define SYS_BRBCR_EL12 sys_reg(2, 5, 9, 0, 0) #define SYS_TTBR0_EL12 sys_reg(3, 5, 2, 0, 0) #define SYS_TTBR1_EL12 sys_reg(3, 5, 2, 0, 1) -#define SYS_SPSR_EL12 sys_reg(3, 5, 4, 0, 0) #define SYS_ELR_EL12 sys_reg(3, 5, 4, 0, 1) #define SYS_AFSR0_EL12 sys_reg(3, 5, 5, 1, 0) #define SYS_AFSR1_EL12 sys_reg(3, 5, 5, 1, 1) @@ -826,6 +829,8 @@ #define SCTLR_ELx_A (BIT(1)) #define SCTLR_ELx_M (BIT(0)) +#endif /* ARM64_S390_COMMON == 1 */ + #define INIT_SCTLR_EL2_MMU_ON \ (SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \ SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_ITFSB | \ @@ -847,6 +852,7 @@ SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS | SCTLR_EL1_TSCXT | \ SCTLR_EL1_EOS) +#if ARM64_S390_COMMON == 1 /* MAIR_ELx memory attributes (used by Linux) */ #define MAIR_ATTR_DEVICE_nGnRnE UL(0x00) #define MAIR_ATTR_DEVICE_nGnRE UL(0x04) @@ -1018,7 +1024,7 @@ #define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \ GCS_CAP_VALID_TOKEN) /* - * Definitions for GICv5 instructions + * Definitions for GICv5 instructions for the Current Domain */ #define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3) #define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0) @@ -1083,6 +1089,19 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +/* + * Definitions for GICv5 instructions for the Virtual Domain + */ +#define GICV5_OP_GIC_VDPEND sys_insn(1, 4, 12, 1, 4) + +/* Shift and mask definitions for GIC VDPEND */ +#define GICV5_GIC_VDPEND_PENDING_MASK BIT_ULL(63) +#define GICV5_GIC_VDPEND_VM_MASK GENMASK_ULL(47, 32) +#define GICV5_GIC_VDPEND_TYPE_MASK GENMASK_ULL(31, 29) +#define GICV5_GIC_VDPEND_ID_MASK GENMASK_ULL(23, 0) + +#endif /* ARM64_S390_COMMON == 1 */ + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) @@ -1232,6 +1251,8 @@ par; \ }) +#if ARM64_S390_COMMON == 1 + #define SYS_FIELD_VALUE(reg, field, val) reg##_##field##_##val #define SYS_FIELD_GET(reg, field, val) \ @@ -1244,6 +1265,8 @@ FIELD_PREP(reg##_##field##_MASK, \ SYS_FIELD_VALUE(reg, field, val)) -#endif +#endif /* ARM64_S390_COMMON == 1 */ + +#endif /* __ASSEMBLER__ */ #endif /* __ASM_SYSREG_H */
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 019e5e3..65ee09b 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -97,6 +97,11 @@ struct kvm_regs { #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) #define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) +/* Supported VGICv5 address types */ +#define KVM_VGIC_V5_ADDR_TYPE_IRS 6 + +#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ @@ -418,6 +423,16 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 +#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 +#define KVM_DEV_ARM_VGIC_GRP_IST 11 + +struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; +}; + #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index facda81..19d602e 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h
@@ -142,6 +142,7 @@ KVM_NVHE_ALIAS(__hyp_rodata_end); #ifdef CONFIG_NVHE_EL2_TRACING KVM_NVHE_ALIAS(__hyp_event_ids_start); KVM_NVHE_ALIAS(__hyp_event_ids_end); +KVM_NVHE_ALIAS(__hyp_string_fmts_start); #endif /* pKVM static key */
diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile index 4a0cbed..a273ab5 100644 --- a/arch/arm64/kernel/pi/Makefile +++ b/arch/arm64/kernel/pi/Makefile
@@ -34,6 +34,7 @@ $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) obj-y := idreg-override.pi.o \ map_kernel.pi.o map_range.pi.o \
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index af1d720..1869b44 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -15,7 +15,7 @@ #ifdef CONFIG_NVHE_EL2_TRACING #define HYPERVISOR_EVENT_IDS \ - . = ALIGN(PAGE_SIZE); \ + . = ALIGN(SZ_8); \ __hyp_event_ids_start = .; \ *(HYP_SECTION_NAME(.event_ids)) \ __hyp_event_ids_end = .; @@ -214,6 +214,14 @@ HYPERVISOR_RODATA_SECTIONS +#ifdef CONFIG_NVHE_EL2_TRACING + .rodata.hyp_string_fmts : { + __hyp_string_fmts_start = .; + *(_hyp_string_fmts) + __hyp_string_fmts_end = .; + } +#endif + .got : { *(.got) } /* * Make sure that the .got.plt is either completely empty or it
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449154f..48d531e 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig
@@ -22,21 +22,18 @@ select KVM_COMMON select KVM_GENERIC_HARDWARE_ENABLING select HAVE_KVM_CPU_RELAX_INTERCEPT - select KVM_MMIO select KVM_GENERIC_DIRTYLOG_READ_PROTECT select VIRT_XFER_TO_GUEST_WORK select KVM_VFIO select HAVE_KVM_DIRTY_RING_ACQ_REL select NEED_KVM_DIRTY_RING_WITH_BITMAP - select HAVE_KVM_MSI select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQ_BYPASS - select HAVE_KVM_READONLY_MEM - select HAVE_KVM_VCPU_RUN_PID_CHANGE select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS select KVM_GUEST_MEMFD + select KVM_GENERIC_PRE_FAULT_MEMORY help Support hosting virtualized guest machines.
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 59612d2..7aaeeb8 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile
@@ -24,7 +24,8 @@ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \ - vgic/vgic-v5.o + vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o \ + vgic-sys-reg-v5.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o
diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321..b2c5ba4 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c
@@ -528,17 +528,11 @@ static void timer_save_state(struct arch_timer_context *ctx) goto out; switch (index) { - u64 cval; - case TIMER_VTIMER: case TIMER_HVTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL)); - cval = read_sysreg_el0(SYS_CNTV_CVAL); - - if (has_broken_cntvoff()) - cval -= timer_get_offset(ctx); - - timer_set_cval(ctx, cval); + if (!has_broken_cntvoff() || !timer_get_offset(ctx)) + timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL)); /* Disable the timer */ write_sysreg_el0(0, SYS_CNTV_CTL); @@ -564,11 +558,12 @@ static void timer_save_state(struct arch_timer_context *ctx) case TIMER_PTIMER: case TIMER_HPTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL)); - cval = read_sysreg_el0(SYS_CNTP_CVAL); - - cval -= timer_get_offset(ctx); - - timer_set_cval(ctx, cval); + /* + * With an offset, memory already holds the guest's CVAL (the + * trap handler or __deactivate_traps() wrote it). + */ + if (!timer_get_offset(ctx)) + timer_set_cval(ctx, read_sysreg_el0(SYS_CNTP_CVAL)); /* Disable the timer */ write_sysreg_el0(0, SYS_CNTP_CTL); @@ -647,7 +642,8 @@ static void timer_restore_state(struct arch_timer_context *ctx) offset = timer_get_offset(ctx); if (has_broken_cntvoff()) { set_cntvoff(0); - cval += offset; + if (offset) + cval = timer_apply_offset(cval, offset, kvm_phys_timer_read()); } else { set_cntvoff(offset); } @@ -660,7 +656,8 @@ static void timer_restore_state(struct arch_timer_context *ctx) cval = timer_get_cval(ctx); offset = timer_get_offset(ctx); set_cntpoff(offset); - cval += offset; + if (offset) + cval = timer_apply_offset(cval, offset, kvm_phys_timer_read()); write_sysreg_el0(cval, SYS_CNTP_CVAL); isb(); write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL); @@ -1110,8 +1107,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) timer_context_init(vcpu, i); /* Synchronize offsets across timers of a VM if not already provided */ - if (!vcpu_is_protected(vcpu) && - !test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { + if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read()); timer_set_offset(vcpu_ptimer(vcpu), 0); } @@ -1133,6 +1129,15 @@ void kvm_timer_init_vm(struct kvm *kvm) */ for (int i = 0; i < NR_KVM_TIMERS; i++) kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]); + + /* + * Protected VMs don't allow the userspace to set counter offsets, + * either set via counter register writes or the dedicated ioctls. + * Pretend the offset has already been set and rely on the default + * offset being 0. + */ + if (kvm_vm_is_protected(kvm)) + set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags); } void kvm_timer_cpu_up(void)
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index f2478b5..d30378a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c
@@ -93,6 +93,7 @@ static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = { { KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC }, }; +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu); /* * Set *ext to the capability. * Return 0 if found, or -EINVAL if no IOCTL matches. @@ -214,6 +215,26 @@ static int kvm_arm_default_max_vcpus(void) return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; } +static int kvm_init_vm_flavor(struct kvm *kvm, unsigned long type) +{ + bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; + + if (is_protected_kvm_enabled()) { + if (protected) + kvm->arch.vm_flavor = VM_PROTECTED_PKVM; + else + kvm->arch.vm_flavor = VM_PKVM; + } else if (protected) { + return -EINVAL; + } else if (has_vhe()) { + kvm->arch.vm_flavor = VM_VHE; + } else { + kvm->arch.vm_flavor = VM_NVHE; + } + + return 0; +} + /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -236,6 +257,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif + ret = kvm_init_vm_flavor(kvm, type); + if (ret) + return ret; + ret = kvm_share_hyp(kvm, kvm + 1); if (ret) return ret; @@ -259,12 +284,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) * If any failures occur after this is successful, make sure to * call __pkvm_unreserve_vm to unreserve the VM in hyp. */ - ret = pkvm_init_host_vm(kvm, type); + ret = pkvm_init_host_vm(kvm); if (ret) goto err_uninit_mmu; - } else if (type & KVM_VM_TYPE_ARM_PROTECTED) { - ret = -EINVAL; - goto err_uninit_mmu; } kvm_vgic_early_init(kvm); @@ -410,6 +432,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_COUNTER_OFFSET: case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: case KVM_CAP_ARM_SEA_TO_USER: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: @@ -451,7 +474,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = system_supports_mte(); break; case KVM_CAP_STEAL_TIME: - r = kvm_arm_pvtime_supported(); + r = kvm_arm_pvtime_supported(kvm); break; case KVM_CAP_ARM_EL1_32BIT: r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); @@ -552,6 +575,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) mutex_unlock(&vcpu->mutex); #endif + kvm_init_vcpu_ops(vcpu); + /* Force users to call KVM_ARM_VCPU_INIT */ vcpu_clear_flag(vcpu, VCPU_INITIALIZED); @@ -595,7 +620,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) if (!is_protected_kvm_enabled()) kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); else - free_hyp_memcache(&vcpu->arch.pkvm_memcache); + free_hyp_memcache(&vcpu->arch.stage2_mc); kvm_timer_vcpu_terminate(vcpu); kvm_pmu_vcpu_destroy(vcpu); kvm_vgic_vcpu_destroy(vcpu); @@ -614,33 +639,34 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) { - if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) { - /* - * Either we're running an L2 guest, and the API/APK bits come - * from L1's HCR_EL2, or API/APK are both set. - */ - if (unlikely(is_nested_ctxt(vcpu))) { - u64 val; + if (!vcpu_has_ptrauth(vcpu)) + return; - val = __vcpu_sys_reg(vcpu, HCR_EL2); - val &= (HCR_API | HCR_APK); - vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); - vcpu->arch.hcr_el2 |= val; - } else { - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); - } + /* + * Either we're running an L2 guest, and the API/APK bits come + * from L1's HCR_EL2, or API/APK are both set. + */ + if (unlikely(is_nested_ctxt(vcpu))) { + u64 val; - /* - * Save the host keys if there is any chance for the guest - * to use pauth, as the entry code will reload the guest - * keys in that case. - */ - if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { - struct kvm_cpu_context *ctxt; + val = __vcpu_sys_reg(vcpu, HCR_EL2); + val &= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= val; + } else { + vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + } - ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); - ptrauth_save_keys(ctxt); - } + /* + * Save the host keys if there is any chance for the guest + * to use pauth, as the entry code will reload the guest + * keys in that case. + */ + if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { + struct kvm_cpu_context *ctxt; + + ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); + ptrauth_save_keys(ctxt); } } @@ -666,14 +692,11 @@ static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu) return single_task_running(); } -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vcpu_prepare_mmu(struct kvm_vcpu *vcpu) { struct kvm_s2_mmu *mmu; int *last_ran; - if (is_protected_kvm_enabled()) - goto nommu; - if (vcpu_has_nv(vcpu)) kvm_vcpu_load_hw_mmu(vcpu); @@ -703,25 +726,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_call_hyp(__kvm_flush_cpu_context, mmu); *last_ran = vcpu->vcpu_idx; } +} -nommu: - vcpu->cpu = cpu; +static void vcpu_put_mmu(struct kvm_vcpu *vcpu) +{ + kvm_arm_vmid_clear_active(); +} - /* - * The timer must be loaded before the vgic to correctly set up physical - * interrupt deactivation in nested state (e.g. timer interrupt). - */ - kvm_timer_vcpu_load(vcpu); - kvm_vgic_load(vcpu); - kvm_vcpu_load_debug(vcpu); - kvm_vcpu_load_fgt(vcpu); - if (has_vhe()) - kvm_vcpu_load_vhe(vcpu); - kvm_arch_vcpu_load_fp(vcpu); - kvm_vcpu_pmu_restore_guest(vcpu); - if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) - kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); - +static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) +{ if (kvm_vcpu_should_clear_twe(vcpu)) vcpu->arch.hcr_el2 &= ~HCR_TWE; else @@ -731,16 +744,73 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.hcr_el2 &= ~HCR_TWI; else vcpu->arch.hcr_el2 |= HCR_TWI; +} +static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) +{ + if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +} + +static void vhe_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu_prepare_mmu(vcpu); + /* + * For VHE, the timer must be loaded before the vgic to correctly + * set up physical interrupt deactivation in nested state (e.g. timer + * interrupt). + */ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_vcpu_load_vhe(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); vcpu_set_pauth_traps(vcpu); +} - if (is_protected_kvm_enabled()) { - kvm_call_hyp_nvhe(__pkvm_vcpu_load, - vcpu->kvm->arch.pkvm.handle, - vcpu->vcpu_idx, vcpu->arch.hcr_el2); - kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, - &vcpu->arch.vgic_cpu.vgic_v3); - } +static void nvhe_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu_prepare_mmu(vcpu); + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + vcpu_set_pauth_traps(vcpu); +} + +static void pkvm_vcpu_load(struct kvm_vcpu *vcpu) +{ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + + kvm_call_hyp_nvhe(__pkvm_vcpu_load, + vcpu->kvm->arch.pkvm.handle, + vcpu->vcpu_idx, vcpu->arch.hcr_el2); + kvm_call_hyp_nvhe(__vgic_v3_restore_vmcr_aprs, + &vcpu->arch.vgic_cpu.vgic_v3); +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; + vcpu->arch.vcpu_ops->vcpu_load(vcpu); if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) vcpu_set_on_unsupported_cpu(vcpu); @@ -748,49 +818,76 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.pid = pid_nr(vcpu->pid); } -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +static void vhe_vcpu_put(struct kvm_vcpu *vcpu) { - if (is_protected_kvm_enabled()) { - kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); - kvm_call_hyp_nvhe(__pkvm_vcpu_put); - - /* __pkvm_vcpu_put implies a sync of the state */ - if (!kvm_vm_is_protected(vcpu->kvm)) - vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); - } - kvm_vcpu_put_debug(vcpu); kvm_arch_vcpu_put_fp(vcpu); - if (has_vhe()) - kvm_vcpu_put_vhe(vcpu); + kvm_vcpu_put_vhe(vcpu); kvm_timer_vcpu_put(vcpu); kvm_vgic_put(vcpu); kvm_vcpu_pmu_restore_host(vcpu); + if (vcpu_has_nv(vcpu)) kvm_vcpu_put_hw_mmu(vcpu); - kvm_arm_vmid_clear_active(); + vcpu_put_mmu(vcpu); +} + +static void nvhe_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); + vcpu_put_mmu(vcpu); +} + +static void pkvm_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_call_hyp_nvhe(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); + kvm_call_hyp_nvhe(__pkvm_vcpu_put); + + /* __pkvm_vcpu_put implies a sync of the state */ + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + vcpu->arch.vcpu_ops->vcpu_put(vcpu); vcpu_clear_on_unsupported_cpu(vcpu); vcpu->cpu = -1; } -static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) +static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu, u32 mp_state) { - WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + WRITE_ONCE(vcpu->arch.mp_state.mp_state, mp_state); kvm_make_request(KVM_REQ_SLEEP, vcpu); kvm_vcpu_kick(vcpu); } +/* The guest's own CPU_OFF: EL2 has already powered the vCPU off. */ void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) { + u32 mp_state = vcpu_is_protected(vcpu) ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_STOPPED; + spin_lock(&vcpu->arch.mp_state_lock); - __kvm_arm_vcpu_power_off(vcpu); + __kvm_arm_vcpu_power_off(vcpu, mp_state); spin_unlock(&vcpu->arch.mp_state_lock); } bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) { - return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; + u32 mp_state = READ_ONCE(vcpu->arch.mp_state.mp_state); + + return mp_state == KVM_MP_STATE_STOPPED || mp_state == KVM_MP_STATE_UNINITIALIZED; } static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu) @@ -805,6 +902,12 @@ static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu) return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED; } +/* Only the guest's CPU_ON may start a vCPU EL2 holds powered off. */ +bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu) +{ + return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_UNINITIALIZED; +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -822,12 +925,24 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, switch (mp_state->mp_state) { case KVM_MP_STATE_RUNNABLE: + if (kvm_pkvm_vcpu_is_powered_off(vcpu)) { + ret = -EPERM; + break; + } + WRITE_ONCE(vcpu->arch.mp_state, *mp_state); break; case KVM_MP_STATE_STOPPED: - __kvm_arm_vcpu_power_off(vcpu); + /* Keep EL2's off record: STOPPED over it would let RUNNABLE through. */ + if (!kvm_pkvm_vcpu_is_powered_off(vcpu)) + __kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED); break; case KVM_MP_STATE_SUSPENDED: + if (kvm_pkvm_vcpu_is_powered_off(vcpu)) { + ret = -EPERM; + break; + } + kvm_arm_vcpu_suspend(vcpu); break; default: @@ -953,7 +1068,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = kvm_finalize_sys_regs(vcpu); + ret = kvm_vcpu_finalize_sys_regs(vcpu); if (ret) return ret; @@ -983,13 +1098,13 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = vgic_v5_finalize_ppi_state(kvm); + ret = vgic_v5_finalize_ppi_state(vcpu); if (ret) return ret; if (is_protected_kvm_enabled()) { /* Start with the vcpu in a dirty state */ - if (!kvm_vm_is_protected(vcpu->kvm)) + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); ret = pkvm_create_hyp_vm(kvm); if (ret) @@ -1163,6 +1278,14 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) preempt_enable(); } + if (kvm_check_request(KVM_REQ_RELOAD_GICv5, vcpu)) { + /* The IRS enable bit was changed */ + preempt_disable(); + vgic_v5_put(vcpu); + vgic_v5_load(vcpu); + preempt_enable(); + } + if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) kvm_vcpu_reload_pmu(vcpu); @@ -1267,7 +1390,7 @@ static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer * - * This function is called through the VCPU_RUN ioctl called from user space. It + * This function is called through the KVM_RUN ioctl called from user space. It * will execute VM code in a loop until the time slice for the process is used * or some emulation is needed from user space in which case the function will * return with return value 0 and with the kvm_run structure filled in with the @@ -1581,10 +1704,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return -EINVAL; } -static unsigned long system_supported_vcpu_features(void) +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) { unsigned long features = KVM_VCPU_VALID_FEATURES; + if (vcpu->kvm->arch.vm_flavor == VM_PROTECTED_PKVM) + kvm_pkvm_vcpu_allowed_features(vcpu->kvm, &features); + if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); @@ -1607,6 +1733,7 @@ static unsigned long system_supported_vcpu_features(void) return features; } +#if ARM64_S390_COMMON == 1 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, const struct kvm_vcpu_init *init) { @@ -1621,7 +1748,12 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, return -ENOENT; } - if (features & ~system_supported_vcpu_features()) + if (features & ~system_supported_vcpu_features(vcpu)) + return -EINVAL; + + /* EL2 implements PSCI 1.1; the host must not dispatch as 0.1. */ + if (vcpu_is_protected(vcpu) && + !test_bit(KVM_ARM_VCPU_PSCI_0_2, &features)) return -EINVAL; /* @@ -1660,6 +1792,8 @@ static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu, KVM_VCPU_MAX_FEATURES); } +#endif /* ARM64_S390_COMMON == 1 */ + static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; @@ -1779,7 +1913,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, spin_lock(&vcpu->arch.mp_state_lock); if (power_off) - __kvm_arm_vcpu_power_off(vcpu); + __kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED); else WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); @@ -1856,6 +1990,24 @@ static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, return __kvm_arm_vcpu_set_events(vcpu, events); } +/* + * Once a protected vCPU has run, the host copy holds the boot state the VMM + * wrote plus what the exit handlers copy out, and EL2 has read mp_state, + * which it does only at hyp vCPU creation. + */ +static long pkvm_filter_vcpu_ioctl(struct kvm_vcpu *vcpu, unsigned int ioctl) +{ + switch (ioctl) { + case KVM_ARM_VCPU_INIT: + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: + if (vcpu_is_protected(vcpu) && vcpu_has_run_once(vcpu)) + return -EPERM; + } + + return 0; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1864,6 +2016,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, struct kvm_device_attr attr; long r; + r = pkvm_filter_vcpu_ioctl(vcpu, ioctl); + if (r) + return r; + switch (ioctl) { case KVM_ARM_VCPU_INIT: { struct kvm_vcpu_init init; @@ -2109,6 +2265,37 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) } } +static const struct kvm_vcpu_ops vhe_vcpu_ops = { + .vcpu_load = vhe_vcpu_load, + .vcpu_put = vhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops nvhe_vcpu_ops = { + .vcpu_load = nvhe_vcpu_load, + .vcpu_put = nvhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops pkvm_vcpu_ops = { + .vcpu_load = pkvm_vcpu_load, + .vcpu_put = pkvm_vcpu_put, +}; + +#define KVM_VCPU_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vcpu_ops *arm64_vcpu_ops[] = { + KVM_VCPU_OPS(VM_NVHE, nvhe_vcpu_ops), + KVM_VCPU_OPS(VM_VHE, vhe_vcpu_ops), + KVM_VCPU_OPS(VM_PKVM, pkvm_vcpu_ops), + KVM_VCPU_OPS(VM_PROTECTED_PKVM, pkvm_vcpu_ops), +}; + +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vcpu_ops) != VM_FLAVOR_MAX); + vcpu->arch.vcpu_ops = arm64_vcpu_ops[vcpu->kvm->arch.vm_flavor]; +} + static unsigned long nvhe_percpu_size(void) { return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - @@ -2626,6 +2813,10 @@ static void kvm_hyp_init_symbols(void) kvm_nvhe_sym(__icache_flags) = __icache_flags; kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits; + kvm_nvhe_sym(spectre_v2_state) = arm64_get_spectre_v2_state(); + kvm_nvhe_sym(spectre_v4_state) = arm64_get_spectre_v4_state(); + kvm_nvhe_sym(spectre_bhb_state) = arm64_get_spectre_bhb_state(); + /* Propagate the FGT state to the nVHE side */ kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks; kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks; @@ -2918,6 +3109,8 @@ static int __init init_hyp_mode(void) kvm_err("Failed to init hyp memory protection\n"); goto out_err; } + + pkvm_selftests(); } return 0;
diff --git a/arch/arm64/include/asm/kvm_define_hypevents.h b/arch/arm64/kvm/define_hypevents.h similarity index 100% rename from arch/arm64/include/asm/kvm_define_hypevents.h rename to arch/arm64/kvm/define_hypevents.h
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index b01d662..1079136 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c
@@ -62,6 +62,7 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { sizeof(kvm_vcpu_stats_desc), }; +#if ARM64_S390_COMMON == 1 static bool core_reg_offset_is_vreg(u64 off) { return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && @@ -134,19 +135,19 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) KVM_REG_ARM_CORE_REG(regs.regs[30]): off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); off /= 2; - return &vcpu->arch.ctxt.regs.regs[off]; + return &vcpu_gp_regs(vcpu)[off]; case KVM_REG_ARM_CORE_REG(regs.sp): - return &vcpu->arch.ctxt.regs.sp; + return vcpu_sp_el0(vcpu); case KVM_REG_ARM_CORE_REG(regs.pc): - return &vcpu->arch.ctxt.regs.pc; + return vcpu_pc(vcpu); case KVM_REG_ARM_CORE_REG(regs.pstate): - return &vcpu->arch.ctxt.regs.pstate; + return vcpu_cpsr(vcpu); case KVM_REG_ARM_CORE_REG(sp_el1): - return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); + return vcpu_sp_el1(vcpu); case KVM_REG_ARM_CORE_REG(elr_el1): return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); @@ -170,13 +171,13 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); off /= 4; - return &vcpu->arch.ctxt.fp_regs.vregs[off]; + return vcpu_vreg(vcpu, off); case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): - return &vcpu->arch.ctxt.fp_regs.fpsr; + return vcpu_fpsr(vcpu); case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): - return &vcpu->arch.ctxt.fp_regs.fpcr; + return vcpu_fpcr(vcpu); default: return NULL; @@ -306,6 +307,8 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return err; } +#endif /* ARM64_S390_COMMON == 1 */ + #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) @@ -543,6 +546,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) return -EINVAL; } +#if ARM64_S390_COMMON == 1 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, u64 __user *uindices) { @@ -591,6 +595,8 @@ static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) return copy_core_reg_indices(vcpu, NULL); } +#endif /* ARM64_S390_COMMON == 1 */ + static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) { const unsigned int slices = vcpu_sve_slices(vcpu); @@ -787,6 +793,13 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, int ret = 0; /* + * EL2 injects an external abort only to complete a forwarded abort. + * SError injection is forwarded. + */ + if (vcpu_is_protected(vcpu) && ext_dabt_pending) + return -EPERM; + + /* * Immediately commit the pending SEA to the vCPU's architectural * state which is necessary since we do not return a pending SEA * to userspace via KVM_GET_VCPU_EVENTS. @@ -883,6 +896,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, { trace_kvm_set_guest_debug(vcpu, dbg->control); + /* A protected guest's debug state is not exposed to the host. */ + if (vcpu_is_protected(vcpu)) + return -EPERM; + if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) return -EINVAL;
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 308100e..9709905 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S
@@ -46,7 +46,7 @@ // Ensure that __guest_enter() always provides a context // synchronization event so that callers don't need ISBs for anything - // that would usually be synchonized by the ERET. + // that would usually be synchronized by the ERET. isb mov x0, #ARM_EXCEPTION_IRQ ret
diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 754e2dc..7521bdd 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c
@@ -70,6 +70,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, enum exception_type type) { unsigned long sctlr, vbar, old, new, mode; + struct kvm *kvm; u64 exc_offset; mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); @@ -109,8 +110,10 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, new |= (old & PSR_C_BIT); new |= (old & PSR_V_BIT); - if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) + kvm = vcpu_get_kvm(vcpu); + if (kvm && kvm_has_mte(kvm)) new |= PSR_TCO_BIT; + vcpu_put_kvm(vcpu, kvm); new |= (old & PSR_DIT_BIT); @@ -276,12 +279,12 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) switch(mode) { case PSR_AA32_MODE_ABT: __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); - vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; + vcpu_gp_regs(vcpu)[__compat_lr_abt] = return_address; break; case PSR_AA32_MODE_UND: __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); - vcpu_gp_regs(vcpu)->compat_lr_und = return_address; + vcpu_gp_regs(vcpu)[__compat_lr_und] = return_address; break; } @@ -350,12 +353,19 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu) */ void __kvm_adjust_pc(struct kvm_vcpu *vcpu) { - if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) { - kvm_inject_exception(vcpu); - vcpu_clear_flag(vcpu, PENDING_EXCEPTION); - vcpu_clear_flag(vcpu, EXCEPT_MASK); - } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) { - kvm_skip_instr(vcpu); - vcpu_clear_flag(vcpu, INCREMENT_PC); + struct kvm_vcpu *target = kvm_adjust_pc_get(vcpu); + + if (!target) + return; + + if (vcpu_get_flag(target, PENDING_EXCEPTION)) { + kvm_inject_exception(target); + vcpu_clear_flag(target, PENDING_EXCEPTION); + vcpu_clear_flag(target, EXCEPT_MASK); + } else if (vcpu_get_flag(target, INCREMENT_PC)) { + kvm_skip_instr(target); + vcpu_clear_flag(target, INCREMENT_PC); } + + kvm_adjust_pc_put(vcpu, target); }
diff --git a/arch/arm64/kvm/hyp/hyp-constants.c b/arch/arm64/kvm/hyp/hyp-constants.c index b257a3b..b2caae2 100644 --- a/arch/arm64/kvm/hyp/hyp-constants.c +++ b/arch/arm64/kvm/hyp/hyp-constants.c
@@ -7,7 +7,5 @@ int main(void) { DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page)); - DEFINE(PKVM_HYP_VM_SIZE, sizeof(struct pkvm_hyp_vm)); - DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu)); return 0; }
diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h index 4fdfeab..d2b9a98 100644 --- a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h +++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
@@ -13,6 +13,34 @@ #include <asm/kvm_emulate.h> #include <asm/kvm_host.h> +#ifdef __KVM_NVHE_HYPERVISOR__ +#include <nvhe/pkvm.h> + +/* Under pKVM a host vCPU's ->kvm is host-writable. */ +static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + if (is_protected_kvm_enabled()) + return pkvm_vcpu_get_kvm(vcpu); + + return kern_hyp_va(vcpu->kvm); +} + +static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ + if (is_protected_kvm_enabled()) + pkvm_vcpu_put_kvm(vcpu, kvm); +} +#else +static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm; +} + +static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ +} +#endif + static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) { if (vcpu_mode_is_32bit(vcpu)) { @@ -33,11 +61,11 @@ static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) static inline void __kvm_skip_instr(struct kvm_vcpu *vcpu) { *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); - vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR); + *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); kvm_skip_instr(vcpu); - write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR); + write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); } @@ -50,4 +78,24 @@ static inline void kvm_skip_host_instr(void) write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR); } +/* + * Under pKVM, the vCPU __kvm_adjust_pc() adjusts for @vcpu (NULL leaves the + * request on @vcpu for its next entry), and the copy of the consumed + * PC_UPDATE_REQ back to @vcpu. + */ +#ifdef __KVM_NVHE_HYPERVISOR__ +struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu); +void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target); +#else +static inline struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) +{ + return vcpu; +} + +static inline void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, + struct kvm_vcpu *target) +{ +} +#endif + #endif
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 1ce7130..a2a054f 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -470,7 +470,7 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) u64 spsr; *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); - arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); + arm64_mops_reset_regs(&vcpu->arch.ctxt.regs, vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* @@ -902,7 +902,7 @@ static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu) /* * Check for the conditions of Cortex-A510's #2077057. When these occur * SPSR_EL2 can't be trusted, but isn't needed either as it is - * unchanged from the value in vcpu_gp_regs(vcpu)->pstate. + * unchanged from the value in vcpu_cpsr(vcpu). * Are we single-stepping the guest, and took a PAC exception from the * active-not-pending state? */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/alloc.h b/arch/arm64/kvm/hyp/include/nvhe/alloc.h new file mode 100644 index 0000000..bca0168 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/alloc.h
@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_NVHE_ALLOC__ +#define __KVM_NVHE_ALLOC__ +#include <linux/types.h> + +#include <asm/kvm_host.h> + +void *hyp_alloc(size_t size); +int hyp_alloc_errno(void); +u32 hyp_alloc_topup_needed(void); +void hyp_free(void *addr); + +int hyp_alloc_init(size_t size); +int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc); +unsigned long hyp_alloc_reclaimable(void); +void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); + +#ifdef CONFIG_NVHE_EL2_DEBUG +int hyp_allocator_selftest(void); +u32 hyp_alloc_selftest_topup_needed(void); +int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc); +void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); +#else +static inline int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) { return 0; }; +static inline void +hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) { }; +#endif +#endif
diff --git a/arch/arm64/kvm/hyp/include/nvhe/define_events.h b/arch/arm64/kvm/hyp/include/nvhe/define_events.h index 776d4c6..370e8c2 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/define_events.h +++ b/arch/arm64/kvm/hyp/include/nvhe/define_events.h
@@ -10,5 +10,3 @@ #define HYP_EVENT_MULTI_READ #include <asm/kvm_hypevents.h> #undef HYP_EVENT_MULTI_READ - -#undef HYP_EVENT
diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 31cae95..72b7927 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h
@@ -25,11 +25,14 @@ int hyp_back_vmemmap(phys_addr_t back); int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot); +void pkvm_remove_mappings(void *from, void *to); int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); int pkvm_check_host_ownership(void); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); +int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size); +phys_addr_t pkvm_private_va_range_pa(void *va); #endif /* __KVM_HYP_MM_H */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 5c050f2..4373e9b 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
@@ -7,6 +7,7 @@ #ifndef __ARM64_KVM_NVHE_PKVM_H__ #define __ARM64_KVM_NVHE_PKVM_H__ +#include <asm/kvm_hcall.h> #include <asm/kvm_pkvm.h> #include <nvhe/gfp.h> @@ -35,6 +36,15 @@ struct pkvm_hyp_vcpu { * per-cpu pointer tracking us. Otherwise, NULL if not loaded. */ struct pkvm_hyp_vcpu **loaded_hyp_vcpu; + + /* The previous exit's ARM_EXCEPTION_* code. */ + u32 exit_code; + + /* + * PSCI_0_2_AFFINITY_LEVEL_{OFF, ON_PENDING, ON}. A non-protected + * vCPU is always ON. + */ + int power_state; }; /* @@ -50,8 +60,15 @@ struct pkvm_hyp_vm { struct kvm_pgtable pgt; struct kvm_pgtable_mm_ops mm_ops; struct hyp_pool pool; + unsigned short refcount; hyp_spinlock_t lock; + /* + * The vCPU initialised RUNNABLE: claimed under vm_table_lock, + * released only if its own init fails. + */ + struct pkvm_hyp_vcpu *primary_vcpu; + /* Array of the hyp vCPU structures for this VM. */ struct pkvm_hyp_vcpu *vcpus[]; }; @@ -74,14 +91,24 @@ static inline bool pkvm_hyp_vm_is_protected(struct pkvm_hyp_vm *hyp_vm) return kvm_vm_is_protected(&hyp_vm->kvm); } +static inline void pkvm_hyp_vm_ref_inc(struct pkvm_hyp_vm *hyp_vm) +{ + BUG_ON(hyp_vm->refcount == USHRT_MAX); + hyp_vm->refcount++; +} + +static inline void pkvm_hyp_vm_ref_dec(struct pkvm_hyp_vm *hyp_vm) +{ + BUG_ON(!hyp_vm->refcount); + hyp_vm->refcount--; +} + void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); void __pkvm_unreserve_vm(pkvm_handle_t handle); -int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva); -int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - unsigned long vcpu_hva); +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva); +int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn); int __pkvm_start_teardown_vm(pkvm_handle_t handle); @@ -92,6 +119,8 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx); void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void); +struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu); +void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm); struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle); struct pkvm_hyp_vm *get_np_pkvm_hyp_vm(pkvm_handle_t handle); @@ -101,6 +130,9 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code); void kvm_init_pvm_id_regs(struct kvm_vcpu *vcpu); +void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu); int kvm_check_pvm_sysreg_table(void); +int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); +struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *vm, unsigned long mpidr); #endif /* __ARM64_KVM_NVHE_PKVM_H__ */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h index b6978f0..4116c73 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h +++ b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h
@@ -17,6 +17,8 @@ #include <asm/lse.h> #include <asm/rwonce.h> +#include <linux/cleanup.h> + typedef union hyp_spinlock { u32 __val; struct { @@ -118,4 +120,6 @@ static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { } #endif +DEFINE_LOCK_GUARD_1(hyp_spinlock, hyp_spinlock_t, hyp_spin_lock(_T->lock), + hyp_spin_unlock(_T->lock)) #endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index 8813ff2..6e3225d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h
@@ -4,6 +4,7 @@ #include <linux/trace_remote_event.h> +#include <asm/kvm_hcall.h> #include <asm/kvm_hyptrace.h> static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) @@ -46,7 +47,71 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) void *tracing_reserve_entry(unsigned long length); void tracing_commit_entry(void); -int __tracing_load(unsigned long desc_va, size_t desc_size); +/* + * The trace_hyp_printk boilerplate is too fiddly to be declared with + * HYP_EVENT(): + * + * The string format is stored into a kernel-accessible ELF section. The + * hypervisor only writes the format ID. + * + * The function has a variadic prototype. We have no easy way to know each + * argument width so they must all cast to u64. + */ +#define REMOTE_EVENT_CUSTOM_PRINTK(...) + +#define __TO_U64_0() +#define __TO_U64_1(x) , (u64)(x) +#define __TO_U64_2(x, ...) , (u64)(x) __TO_U64_1(__VA_ARGS__) +#define __TO_U64_3(x, ...) , (u64)(x) __TO_U64_2(__VA_ARGS__) +#define __TO_U64_4(x, ...) , (u64)(x) __TO_U64_3(__VA_ARGS__) +#define __TO_U64_5(x, ...) , (u64)(x) __TO_U64_4(__VA_ARGS__) +#define __TO_U64_6(x, ...) , (u64)(x) __TO_U64_5(__VA_ARGS__) +#define __TO_U64_7(x, ...) , (u64)(x) __TO_U64_6(__VA_ARGS__) +#define __TO_U64_8(x, ...) , (u64)(x) __TO_U64_7(__VA_ARGS__) + +#define __TO_U64_X(N, ...) CONCATENATE(__TO_U64_, N)(__VA_ARGS__) +#define __TO_U64(...) __TO_U64_X(COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +REMOTE_EVENT_FORMAT(hyp_printk, HE_STRUCT(he_field(u16, fmt_id) he_field(u64, args[]))); +extern struct hyp_event_id hyp_event_id_hyp_printk; + +static __always_inline void __trace_hyp_printk(const struct hyp_string_fmt *fmt, int nr_args, ...) +{ + struct remote_event_format_hyp_printk *entry; + va_list va; + int i; + + if (!atomic_read(&hyp_event_id_hyp_printk.enabled)) + return; + + entry = tracing_reserve_entry(struct_size(entry, args, nr_args)); + if (!entry) + return; + + entry->hdr.id = hyp_event_id_hyp_printk.id; + entry->fmt_id = fmt - __hyp_string_fmts_start; + + va_start(va, nr_args); + for (i = 0; i < nr_args; i++) + entry->args[i] = va_arg(va, u64); + va_end(va); + + tracing_commit_entry(); +} + +#define trace_hyp_printk(__fmt, __args...) \ +do { \ + static const struct hyp_string_fmt __used __section("_hyp_string_fmts") \ + __hyp_str_fmt = { \ + .fmt = __fmt, \ + .nr_args = COUNT_ARGS(__args), \ + }; \ + BUILD_BUG_ON(sizeof(__fmt) > HYP_STRING_FMT_MAX_SIZE); \ + /* __TO_U64 prepends a comma if there are arguments */ \ + __trace_hyp_printk(&__hyp_str_fmt, COUNT_ARGS(__args) __TO_U64(__args)); \ +} while (0) + +int __tracing_load(void __kern *desc_va, size_t desc_size); void __tracing_unload(void); int __tracing_enable(bool enable); int __tracing_swap_reader(unsigned int cpu); @@ -58,8 +123,10 @@ static inline void *tracing_reserve_entry(unsigned long length) { return NULL; } static inline void tracing_commit_entry(void) { } #define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \ static inline void trace_##__name(__proto) {} +#define REMOTE_EVENT_CUSTOM_PRINTK(...) +#define trace_hyp_printk(fmt, args...) do { } while (0) -static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; } +static inline int __tracing_load(void __kern *desc_va, size_t desc_size) { return -ENODEV; } static inline void __tracing_unload(void) { } static inline int __tracing_enable(bool enable) { return -ENODEV; } static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; }
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 2ed489b..56a7e2c 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -23,7 +23,7 @@ CFLAGS_switch.nvhe.o += -Wno-override-init hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ - hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ + hyp-main.o hyp-smp.o psci-relay.o alloc.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o @@ -49,6 +49,7 @@ # avoids file name clashes for files shared with VHE. $(obj)/%.nvhe.o: $(src)/%.c FORCE $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) $(obj)/%.nvhe.o: $(src)/%.S FORCE $(call if_changed_rule,as_o_S)
diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c new file mode 100644 index 0000000..e1e6dd2 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c
@@ -0,0 +1,1212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2026 Google LLC + * Author: Vincent Donnefort <vdonnefort@google.com> + * + * This heap allocator manages a reserved VA space range, dynamically mapping + * and unmapping physical pages on-demand to minimise the pKVM hypervisor + * footprint. As memory is reclaimed and relinquished to the host, unmapped + * holes are introduced within the VA space. To prevent orphans mapped regions, + * neighboring unused chunks cannot be merged if they are separated by an + * unmapped region. + * + */ + +#include <nvhe/alloc.h> +#include <nvhe/mem_protect.h> +#include <nvhe/mm.h> +#include <nvhe/spinlock.h> + +#include <linux/build_bug.h> +#include <linux/hash.h> + +/* + * A minimum of 16 bytes allows us to comply with the alignment requirement for + * FPSIMD registers that are part of struct pkvm_hyp_vcpu. + * + * It also must be a power of two. + */ +#define MIN_ALLOC_SIZE 16UL + +/* + * struct chunk_hdr - Chunk header + * @next: offset from this chunk header to the next one. + * @prev: offset from this chunk header to the previous one. + * @__unmapped: Internal field containing the offset to the unmapped page + * boundary, multiplexed with the allocation state flag. + * @hash: Hash computed over the chunk header. + * @data: Chunk payload. + */ +struct chunk_hdr { + u32 next; + u32 prev; +#define USED_BIT_MASK 1U + u32 __unmapped; + u32 hash; + char data[]; +} __aligned(MIN_ALLOC_SIZE); + +/* + * struct hyp_allocator - Heap allocator + * @start: Start in the allocator's reserved virtual address range. + * @end: End in the allocator's reserved virtual address range. + * @last_used: Pointer to the end of the last used chunk. This is + * necessary for the last chunk in the list as the + * allocated size of a chunk is derived from the next one. + * @first_unmapped: Pointer to the first unmapped page in the + * allocator's range. This is only necessary and + * updated when no chunk is in the list. + * @head: Head of the chunk list. + * @tail: Tail of the chunk list. + * @mc: Memcache containing pre-allocated pages for mapping. + * @lock: Spinlock protecting the allocator state. + * @errno: Per-CPU error code for allocation failures. + * @topup_needed: Per-CPU page counter needed to top-up the memcache. + */ +struct hyp_allocator { + void *start; + void *end; + void *last_used; + void *first_unmapped; + struct chunk_hdr *head; + struct chunk_hdr *tail; + struct kvm_hyp_memcache mc; + hyp_spinlock_t lock; + int __percpu *errno; + u32 __percpu *topup_needed; +}; + +static u32 chunk_hash_compute(const struct chunk_hdr *chunk) +{ + u32 hash = 0; + + BUILD_BUG_ON(sizeof(*chunk) != 16); + + hash ^= hash_64(*(const u64 *)chunk, 32); + hash ^= hash_32(chunk->__unmapped, 32); + return hash; +} + +static void chunk_set_hash(struct chunk_hdr *chunk) +{ + if (chunk) + chunk->hash = chunk_hash_compute(chunk); +} + +static void chunk_check_hash(const struct chunk_hdr *chunk) +{ + if (chunk) + WARN_ON(chunk->hash != chunk_hash_compute(chunk)); +} + +static bool chunk_is_used(const struct chunk_hdr *chunk) +{ + return !!(chunk->__unmapped & USED_BIT_MASK); +} + +static void chunk_set_used(struct chunk_hdr *chunk) +{ + chunk->__unmapped |= USED_BIT_MASK; +} + +static void chunk_set_unused(struct chunk_hdr *chunk) +{ + chunk->__unmapped &= ~USED_BIT_MASK; +} + +static void *chunk_unmapped(const struct chunk_hdr *chunk) +{ + u32 offset = chunk->__unmapped & ~USED_BIT_MASK; + + if (!offset) + return NULL; + + return (void *)chunk + offset; +} + +static void __chunk_set_unmapped(struct chunk_hdr *chunk, u32 unmapped) +{ + chunk->__unmapped = unmapped | (chunk_is_used(chunk) ? USED_BIT_MASK : 0); +} + +static void chunk_set_unmapped(struct chunk_hdr *chunk, void *unmapped) +{ + WARN_ON(!PAGE_ALIGNED(unmapped)); + + if (unmapped) { + WARN_ON((void *)chunk > unmapped); + __chunk_set_unmapped(chunk, unmapped - (void *)chunk); + } else { + __chunk_set_unmapped(chunk, 0); + } +} + +static void *chunk_data(const struct chunk_hdr *chunk) +{ + return (void *)&chunk->data; +} + +static struct chunk_hdr *__chunk_next(const struct chunk_hdr *chunk) +{ + if (!chunk->next) + return NULL; + + return (struct chunk_hdr *)((void *)chunk + chunk->next); +} + +static struct chunk_hdr *__chunk_prev(const struct chunk_hdr *chunk) +{ + if (!chunk->prev) + return NULL; + + return (struct chunk_hdr *)((void *)chunk - chunk->prev); +} + +static void chunk_set_next(struct chunk_hdr *chunk, struct chunk_hdr *next) +{ + if (!chunk) + return; + + if (next) { + WARN_ON(chunk > next); + chunk->next = (void *)next - (void *)chunk; + } else { + chunk->next = 0; + } +} + +static void chunk_set_prev(struct chunk_hdr *chunk, struct chunk_hdr *prev) +{ + if (!chunk) + return; + + if (prev) { + WARN_ON(chunk < prev); + chunk->prev = (void *)chunk - (void *)prev; + } else { + chunk->prev = 0; + } +} + +static struct chunk_hdr *chunk_get_next(const struct chunk_hdr *chunk) +{ + struct chunk_hdr *next = __chunk_next(chunk); + + chunk_check_hash(next); + return next; +} + +static struct chunk_hdr *chunk_get_prev(const struct chunk_hdr *chunk) +{ + struct chunk_hdr *prev = __chunk_prev(chunk); + + chunk_check_hash(prev); + return prev; +} + +static struct chunk_hdr *chunk_get(struct chunk_hdr *chunk) +{ + chunk_check_hash(chunk); + return chunk; +} + +#define chunk_hdr_size() \ + offsetof(struct chunk_hdr, data) + +#define chunk_min_size() \ + (chunk_hdr_size() + MIN_ALLOC_SIZE) + +static size_t chunk_data_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + void *end; + + if (next) + end = (void *)next; + else + end = allocator->end; + + return end - chunk_data(chunk); +} + +static size_t chunk_mapped_data_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + void *unmapped = chunk_unmapped(chunk); + + if (!unmapped) + return chunk_data_size(chunk, allocator); + + return unmapped - chunk_data(chunk); +} + +static size_t chunk_used_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + + if (!chunk_is_used(chunk)) + return 0; + + if (next) + return chunk_mapped_data_size(chunk, allocator); + + return allocator->last_used - chunk_data(chunk); +} + +static void chunk_list_insert(struct chunk_hdr *chunk, struct chunk_hdr *prev) +{ + struct chunk_hdr *next = NULL; + + WARN_ON(!chunk); + + if (prev) { + next = chunk_get_next(prev); + chunk_set_next(prev, chunk); + chunk_set_hash(prev); + } + + if (next) { + chunk_set_prev(next, chunk); + chunk_set_hash(next); + } + + chunk_set_next(chunk, next); + chunk_set_prev(chunk, prev); +} + +static void chunk_list_del(struct chunk_hdr *chunk) +{ + struct chunk_hdr *prev, *next; + + WARN_ON(!chunk); + + prev = chunk_get_prev(chunk); + next = chunk_get_next(chunk); + + if (prev) { + chunk_set_next(prev, next); + chunk_set_hash(prev); + } + + if (next) { + chunk_set_prev(next, prev); + chunk_set_hash(next); + } +} + +/* + * Return a fixup start address for chunk creation. It makes sure the chunk + * header doesn't cross any page boundary and that it leaves enough space at the + * start of page. This is intended to prevent orphan mapped regions during chunk + * memory reclaim + */ +static void *chunk_start(void *start) +{ + void *page = PTR_ALIGN(start, PAGE_SIZE); + + if (page - start < chunk_hdr_size()) + return page; + + page = PTR_ALIGN_DOWN(start, PAGE_SIZE); + if (start - page < chunk_min_size()) + return page + chunk_min_size(); + + return start; +} + +static void hyp_allocator_set_errno(struct hyp_allocator *allocator, int errno) +{ + *this_cpu_ptr(allocator->errno) = errno; +} + +static int hyp_allocator_errno(struct hyp_allocator *allocator) +{ + int *errno = this_cpu_ptr(allocator->errno); + int ret = *errno; + + *errno = 0; + + return ret; +} + +static int hyp_allocator_map(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + struct chunk_hdr *next, + void *addr, void *end) +{ + void *unmapped = chunk ? chunk_unmapped(chunk) : allocator->first_unmapped; + void *unmapped_end = next ?: allocator->end; + + /* + * hyp_allocator_can_create_chunk() already validates addr/end + * belong to the chunk. + */ + WARN_ON(end <= addr); + + /* The chunk does not span an unmapped region */ + if (!unmapped) + return 0; + + while (unmapped < end) { + void *page = pop_hyp_memcache(&allocator->mc, hyp_phys_to_virt); + int ret; + + if (!page) { + end = PTR_ALIGN(end, PAGE_SIZE); + *this_cpu_ptr(allocator->topup_needed) = + (unsigned long)(end - unmapped) >> PAGE_SHIFT; + return -ENOMEM; + } + + ret = pkvm_map_private_va_range(unmapped, hyp_virt_to_phys(page), PAGE_SIZE); + if (ret) { + push_hyp_memcache(&allocator->mc, page, hyp_virt_to_phys); + return ret; + } + + unmapped += PAGE_SIZE; + + /* + * Reset the unmap field if we've reached the next chunk or the + * allocator boundary. Chunks are always installed to cover a + * mapped region followed by an unmapped one. Therefore, we + * expect next to be PAGE_ALIGNED. + */ + WARN_ON(!PAGE_ALIGNED(unmapped_end)); + if (unmapped == unmapped_end) + unmapped = NULL; + + if (chunk) { + chunk_set_unmapped(chunk, unmapped); + chunk_set_hash(chunk); + } else { + allocator->first_unmapped = unmapped; + } + + if (!unmapped) + break; + } + + return 0; +} + +static void hyp_allocator_unmap(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + void *addr, void *end) +{ + void *unmap = addr; + + /* + * hyp_allocator_chunk_reclaimable() already computes valid addr/end, no + * need to check them again + */ + WARN_ON(end <= addr); + + while (unmap < end) { + phys_addr_t pa = pkvm_private_va_range_pa((void *)unmap); + void *page = hyp_phys_to_virt(pa); + + push_hyp_memcache(&allocator->mc, page, hyp_virt_to_phys); + unmap += PAGE_SIZE; + } + + pkvm_remove_mappings(addr, end); + + if (chunk) { + chunk_set_unmapped(chunk, addr); + chunk_set_hash(chunk); + } else { + allocator->first_unmapped = addr; + } +} + +static bool hyp_allocator_can_create_chunk(struct hyp_allocator *allocator, + const struct chunk_hdr *prev, + const struct chunk_hdr *next, + void *addr, void *end) +{ + void *page, *unmapped; + + if (addr < allocator->start || end > allocator->end) + return false; + + /* First chunk created must be installed at allocator->start */ + if (!prev) + return addr == allocator->start; + + /* Must not overwrite the next chunk */ + if (next && end > (void *)next) + return false; + + /* Must not overwrite the previous chunk */ + if (addr < (chunk_data(prev) + chunk_used_size(prev, allocator))) + return false; + + /* Header must not cross page boundaries */ + page = PTR_ALIGN(addr, PAGE_SIZE); + if (page != addr && (page - addr) < chunk_hdr_size()) + return false; + + /* Must leave a minimum distance from a page-start to maximise reclaim */ + page = PTR_ALIGN_DOWN(addr, PAGE_SIZE); + if (page != addr && (addr - page) < chunk_min_size()) + return false; + + unmapped = chunk_unmapped(prev); + if (!unmapped) + return true; + + /* Must never create an orphan mapped region */ + if (addr > unmapped) + return false; + + return true; +} + +/* + * Tries to create a new chunk in the allocator whose header starts at @addr and + * whose data finishes at @end. + */ +static struct chunk_hdr *hyp_allocator_create_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *prev, void *addr, + void *end, bool used) +{ + struct chunk_hdr *next, *chunk = addr; + void *unmapped; + int ret; + + if (end > allocator->end) + return ERR_PTR(-E2BIG); + + next = prev ? chunk_get_next(prev) : NULL; + if (!hyp_allocator_can_create_chunk(allocator, prev, next, addr, end)) + return ERR_PTR(-EINVAL); + + ret = hyp_allocator_map(allocator, prev, next, addr, end); + if (ret) + return ERR_PTR(ret); + + memset(chunk, 0, sizeof(*chunk)); + if (used) + chunk_set_used(chunk); + else + chunk_set_unused(chunk); + + /* First chunk, first allocation */ + if (!prev) { + chunk_set_unmapped(chunk, allocator->first_unmapped); + chunk_list_insert(chunk, NULL); + chunk_set_hash(chunk); + + allocator->last_used = end; + allocator->head = allocator->tail = chunk; + return chunk; + } + + /* Last chunk in the list */ + if (!next) { + allocator->last_used = end; + allocator->tail = chunk; + } + + /* Inherit prev's unmapped region */ + unmapped = chunk_unmapped(prev); + chunk_set_unmapped(chunk, unmapped); + chunk_list_insert(chunk, prev); + chunk_set_hash(chunk); + + chunk_set_unmapped(prev, NULL); + chunk_set_hash(prev); + + return chunk; +} + +static bool hyp_allocator_can_destroy_chunk(const struct chunk_hdr *prev, + const struct chunk_hdr *next, + const struct chunk_hdr *chunk) +{ + if (chunk_is_used(chunk)) + return false; + + /* Can't merge down unless we are the last one in the list */ + if (next && chunk_is_used(prev)) + return false; + + /* Must never create an orphan mapped region */ + if (chunk_unmapped(prev)) + return false; + + return true; +} + +static int hyp_allocator_destroy_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *prev, + struct chunk_hdr *chunk) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + + if (!hyp_allocator_can_destroy_chunk(prev, next, chunk)) + return -EINVAL; + + /* Last chunk in the list */ + if (!next) { + allocator->last_used = chunk; + allocator->tail = prev; + } + + chunk_set_unmapped(prev, chunk_unmapped(chunk)); + chunk_set_hash(prev); + chunk_list_del(chunk); + + return 0; +} + +/* + * Return the best unused chunk for recycling, that is the smallest chunk + * fitting the allocation which needs to use the least unmapped region. + */ +static struct chunk_hdr *hyp_allocator_find_efficient_chunk(struct hyp_allocator *allocator, + size_t size) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + size_t best_data_size = SIZE_MAX; + size_t best_missing = SIZE_MAX; + + chunk = allocator->head; + while (chunk) { + size_t missing, mapped, data_size; + + if (chunk_is_used(chunk)) + goto next; + + data_size = chunk_data_size(chunk, allocator); + if (data_size < size) + goto next; + + mapped = chunk_mapped_data_size(chunk, allocator); + missing = (size > mapped) ? DIV_ROUND_UP(size - mapped, PAGE_SIZE) : 0; + if (missing > best_missing) + goto next; + + if (missing == best_missing && data_size >= best_data_size) + goto next; + + best_missing = missing; + best_data_size = data_size; + best_chunk = chunk; + +next: + chunk = chunk_get_next(chunk); + } + + return best_chunk; +} + +static struct chunk_hdr *hyp_allocator_reuse_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *chunk, size_t size) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + void *start, *end, *split, *split_end; + int ret; + + start = chunk_data(chunk); + end = start + size; + + /* Last chunk in the list, no need to split */ + if (!next) { + split = split_end = NULL; + allocator->last_used = chunk_data(chunk) + size; + } else { + split = chunk_start(end); + split_end = split + chunk_min_size(); + + if (!hyp_allocator_can_create_chunk(allocator, chunk, next, split, split_end)) + split = split_end = NULL; + } + + /* Batch the mapping of the reused chunk and the split */ + ret = hyp_allocator_map(allocator, chunk, next, chunk_data(chunk), split ? split_end : end); + if (ret) + return ERR_PTR(ret); + + if (split) + WARN_ON(IS_ERR_OR_NULL( + hyp_allocator_create_chunk(allocator, chunk, split, split_end, false))); + + chunk_set_used(chunk); + chunk_set_hash(chunk); + + return chunk; +} + +static struct chunk_hdr *hyp_allocator_alloc_chunk(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk; + void *start, *end; + + /* First allocation */ + if (!allocator->head) { + start = allocator->start; + end = start + chunk_hdr_size() + size; + return hyp_allocator_create_chunk(allocator, NULL, start, end, true); + } + + chunk = hyp_allocator_find_efficient_chunk(allocator, size); + + /* Nothing found, create a new chunk at the end in the list */ + if (!chunk) { + start = chunk_start(chunk_data(allocator->tail) + + chunk_used_size(allocator->tail, allocator)); + end = start + chunk_hdr_size() + size; + return hyp_allocator_create_chunk(allocator, allocator->tail, start, end, true); + } + + return hyp_allocator_reuse_chunk(allocator, chunk, size); +} + +static void *hyp_allocator_alloc(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk; + + size = max(size, MIN_ALLOC_SIZE); + + /* Ensure we do not overflow ALIGN(MIN_ALLOC_SIZE) */ + if (size > U32_MAX) { + hyp_allocator_set_errno(allocator, -E2BIG); + return NULL; + } + + size = ALIGN(size, MIN_ALLOC_SIZE); + if (size > (allocator->end - allocator->start - chunk_hdr_size())) { + hyp_allocator_set_errno(allocator, -E2BIG); + return NULL; + } + + scoped_guard(hyp_spinlock, &allocator->lock) { + chunk = hyp_allocator_alloc_chunk(allocator, size); + if (IS_ERR_OR_NULL(chunk)) { + hyp_allocator_set_errno(allocator, + IS_ERR(chunk) ? PTR_ERR(chunk) : -EINVAL); + return NULL; + } + } + + memset(chunk_data(chunk), 0, size); + return chunk_data(chunk); +} + +static void hyp_allocator_free(struct hyp_allocator *allocator, void *data) +{ + struct chunk_hdr *chunk, *next, *prev; + + if (!data) + return; + + WARN_ON(!IS_ALIGNED((unsigned long)data, MIN_ALLOC_SIZE)); + WARN_ON(data >= allocator->end || data < allocator->start + chunk_hdr_size()); + + guard(hyp_spinlock)(&allocator->lock); + + chunk = chunk_get(container_of(data, struct chunk_hdr, data)); + WARN_ON(!chunk_is_used(chunk)); + chunk_set_unused(chunk); + chunk_set_hash(chunk); + + next = chunk_get_next(chunk); + if (next) + hyp_allocator_destroy_chunk(allocator, chunk, next); + + prev = chunk_get_prev(chunk); + if (prev) + hyp_allocator_destroy_chunk(allocator, prev, chunk); +} + +static unsigned long hyp_allocator_chunk_reclaimable(struct hyp_allocator *allocator, + const struct chunk_hdr *chunk, + u64 *__addr, u64 *__end) +{ + struct chunk_hdr *next; + void *addr, *end; + + /* Last chunk in the allocator */ + if (chunk == allocator->head && chunk == allocator->tail && !chunk_is_used(chunk)) { + addr = (void *)chunk; + end = chunk_unmapped(chunk); + if (!end) + end = allocator->end; + goto end; + } + + next = chunk_get_next(chunk); + + /* Last chunk in the list we can reclaim, even if used */ + if (!next) { + addr = chunk_data(chunk) + chunk_used_size(chunk, allocator); + addr = PTR_ALIGN(addr, PAGE_SIZE); + end = chunk_unmapped(chunk); + if (!end) + end = allocator->end; + goto end; + } + + if (chunk_is_used(chunk)) + return 0; + + addr = PTR_ALIGN(chunk_data(chunk), PAGE_SIZE); + end = chunk_unmapped(chunk); + if (!end) + end = PTR_ALIGN_DOWN(next, PAGE_SIZE); + +end: + if (addr >= end) + return 0; + + if (__end) + *__end = (u64)end; + if (__addr) + *__addr = (u64)addr; + + return (end - addr) >> PAGE_SHIFT; +} + +static void hyp_allocator_reclaim_chunk(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + void *addr, void *end) +{ + struct chunk_hdr *next; + + WARN_ON(end <= addr); + + /* We are about to destroy the last chunk in the allocator */ + if (addr == allocator->start) { + allocator->tail = allocator->head = chunk = NULL; + goto unmap; + } + + next = chunk_get_next(chunk); + + /* + * Split the reclaimed chunk at the next page boundary, + * this ensures no orphan mapped region is created. Splitting at the page boundary is always + * possible because chunks always leave a minimum distance to the page start. + * + * +--------------+ + * |______________| + * |______________|<- Next chunk + * |_ _ _ __ _ _ _| + * | |<- Page-aligned split + * +--------------+ + * +--------------+ + * | | + * | |<- Page reclaimed + * | | + * | | + * +--------------+ + * +--------------+ + * | | + * |______________| + * |______________|<- Chunk to split + * | | + * +--------------+ + */ + if (next && !chunk_unmapped(chunk) && next != end) + WARN_ON(IS_ERR_OR_NULL(hyp_allocator_create_chunk(allocator, chunk, end, next, + false))); +unmap: + hyp_allocator_unmap(allocator, chunk, addr, end); +} + +/* + * Return the best reclaimable chunk which is the highest chunk in the list + * with the biggest reclaimable region. + */ +static struct chunk_hdr *hyp_allocator_find_reclaimable_chunk(struct hyp_allocator *allocator, + u64 *addr, u64 *end) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + unsigned long best_reclaimable = 0; + + chunk = allocator->head; + while (chunk) { + u64 __addr, __end; + unsigned long reclaimable = hyp_allocator_chunk_reclaimable(allocator, chunk, + &__addr, &__end); + + /* Favour the top biggest chunks */ + if (reclaimable && reclaimable >= best_reclaimable) { + best_reclaimable = reclaimable; + best_chunk = chunk; + *addr = __addr; + *end = __end; + } + + chunk = chunk_get_next(chunk); + } + + return best_chunk; +} + +static void hyp_allocator_drain_memcache(struct hyp_allocator *allocator, + struct kvm_hyp_memcache *host_mc, + unsigned long target) +{ + struct kvm_hyp_memcache *mc = &allocator->mc; + + while (target && mc->nr_pages) { + void *page = pop_hyp_memcache(mc, hyp_phys_to_virt); + + memset(page, 0, PAGE_SIZE); + kvm_flush_dcache_to_poc(page, PAGE_SIZE); + push_hyp_memcache(host_mc, page, hyp_virt_to_phys); + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1)); + + target--; + } +} + +static void hyp_allocator_reclaim(struct hyp_allocator *allocator, struct kvm_hyp_memcache *host_mc, + unsigned long target) +{ + unsigned long reclaimed = 0; + + if (!target) + return; + + guard(hyp_spinlock)(&allocator->lock); + + do { + struct chunk_hdr *chunk; + unsigned long r; + u64 addr, end; + + chunk = hyp_allocator_find_reclaimable_chunk(allocator, &addr, &end); + if (!chunk) + break; + + r = min((end - addr) >> PAGE_SHIFT, target - reclaimed); + addr = end - (r << PAGE_SHIFT); + hyp_allocator_reclaim_chunk(allocator, chunk, (void *)addr, (void *)end); + + reclaimed += r; + } while (reclaimed < target); + + hyp_allocator_drain_memcache(allocator, host_mc, target); +} + +static unsigned long hyp_allocator_reclaimable(struct hyp_allocator *allocator) +{ + unsigned long reclaimable; + struct chunk_hdr *chunk; + + guard(hyp_spinlock)(&allocator->lock); + + /* + * While allocator->mc typically contains transient pages, + * over-allocation or an aborted operation could leave pages behind. + * Account for allocator->mc so stalled topup pages can be reclaimed. + */ + reclaimable = allocator->mc.nr_pages; + + chunk = allocator->head; + while (chunk) { + reclaimable += hyp_allocator_chunk_reclaimable(allocator, chunk, NULL, NULL); + chunk = chunk_get_next(chunk); + } + + return reclaimable; +} + +static int hyp_allocator_topup(struct hyp_allocator *allocator, + struct kvm_hyp_memcache *host_mc) +{ + struct kvm_hyp_memcache *alloc_mc = &allocator->mc; + + guard(hyp_spinlock)(&allocator->lock); + return refill_memcache(alloc_mc, host_mc->nr_pages + alloc_mc->nr_pages, host_mc); +} + +static u32 hyp_allocator_topup_needed(struct hyp_allocator *allocator) +{ + u32 *topup_needed = this_cpu_ptr(allocator->topup_needed); + u32 ret = *topup_needed; + + *topup_needed = 0; + + return ret; +} + +static int hyp_allocator_init(struct hyp_allocator *allocator, size_t size) +{ + unsigned long start; + int ret; + + size = PAGE_ALIGN(size); + + /* constrained by chunk_hdr u32 types */ + if (size > U32_MAX || !size) + return -EINVAL; + + ret = pkvm_alloc_private_va_range(size, &start); + if (ret) + return ret; + + allocator->first_unmapped = allocator->start = (void *)start; + allocator->end = allocator->start + size; + hyp_spin_lock_init(&allocator->lock); + + return 0; +} + +static DEFINE_PER_CPU(int, __hyp_allocator_errno); +static DEFINE_PER_CPU(u32, __hyp_allocator_topup_needed); + +static struct hyp_allocator hyp_allocator = { + .errno = &__hyp_allocator_errno, + .topup_needed = &__hyp_allocator_topup_needed, +}; + +/** + * hyp_alloc() - Allocate memory from the heap allocator + * + * @size: Allocation size in bytes. + * + * Callers must ensure they do not hold pkvm_pgd_lock to avoid deadlocks, + * as the allocator may modify the hyp stage-1 page-table. + * + * Return: A pointer to the allocated memory on success, else NULL. + */ +void *hyp_alloc(size_t size) +{ + return hyp_allocator_alloc(&hyp_allocator, size); +} + +/** + * hyp_free() - Free memory allocated with hyp_alloc() + * + * @data: Address returned by the original hyp_alloc(). + * + * The use of any other address than one returned by hyp_alloc() will cause a + * hypervisor panic. + */ +void hyp_free(void *data) +{ + hyp_allocator_free(&hyp_allocator, data); +} + +/** + * hyp_alloc_errno() - Read the errno on allocation error + * + * Get the return code from an allocation failure. + * + * Return: -ENOMEM if the allocator needs a refill from the host, -E2BIG if + * there is no VA space left else 0. + */ +int hyp_alloc_errno(void) +{ + return hyp_allocator_errno(&hyp_allocator); +} + +static int selftest_init(void); + +int hyp_alloc_init(size_t size) +{ + int ret; + + ret = hyp_allocator_init(&hyp_allocator, size); + if (ret) + return ret; + + return selftest_init(); +} + +void hyp_alloc_reclaim(struct kvm_hyp_memcache *mc, unsigned long target) +{ + hyp_allocator_reclaim(&hyp_allocator, mc, target); +} + +unsigned long hyp_alloc_reclaimable(void) +{ + return hyp_allocator_reclaimable(&hyp_allocator); +} + +int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc) +{ + return hyp_allocator_topup(&hyp_allocator, host_mc); +} + +u32 hyp_alloc_topup_needed(void) +{ + return hyp_allocator_topup_needed(&hyp_allocator); +} + +#ifdef CONFIG_NVHE_EL2_DEBUG +#define SELFTEST_MAX_PAGES 6 +#define SELFTEST_MAX_SIZE (PAGE_SIZE * SELFTEST_MAX_PAGES) + +static DEFINE_PER_CPU(int, __selftest_errno); +static DEFINE_PER_CPU(u32, __selftest_topup_needed); + +static struct hyp_allocator selftest_allocator = { + .errno = &__selftest_errno, + .topup_needed = &__selftest_topup_needed, + .lock = __HYP_SPIN_LOCK_UNLOCKED, +}; + +int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) +{ + return hyp_allocator_topup(&selftest_allocator, host_mc); +} + +void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) +{ + hyp_allocator_reclaim(&selftest_allocator, host_mc, target); +} + +u32 hyp_alloc_selftest_topup_needed(void) +{ + return hyp_allocator_topup_needed(&selftest_allocator); +} + +static int selftest_init(void) +{ + return hyp_allocator_init(&selftest_allocator, SELFTEST_MAX_SIZE); +} + +static void *selftest_alloc(size_t size) +{ + return hyp_allocator_alloc(&selftest_allocator, size); +} + +static void selftest_free(void *addr) +{ + hyp_allocator_free(&selftest_allocator, addr); +} + +static int selftest_errno(void) +{ + return hyp_allocator_errno(&selftest_allocator); +} + +int hyp_allocator_selftest(void) +{ + struct hyp_allocator *allocator = &selftest_allocator; + static DEFINE_HYP_SPINLOCK(selftest_lock); + struct kvm_hyp_memcache host_mc = { }; + void *addr1, *addr2, *addr3, *addr4; + int ret; + + guard(hyp_spinlock)(&selftest_lock); + + if (allocator->mc.nr_pages < SELFTEST_MAX_PAGES) { + *this_cpu_ptr(allocator->topup_needed) = SELFTEST_MAX_PAGES - + allocator->mc.nr_pages; + return -ENOMEM; + } + + selftest_alloc(SELFTEST_MAX_SIZE); + if (selftest_errno() != -E2BIG) + return -EINVAL; + + selftest_alloc(SIZE_MAX); + if (selftest_errno() != -E2BIG) + return -EINVAL; + + /* Test first chunk */ + addr1 = selftest_alloc(0); + if (!addr1 || addr1 != (void *)allocator->start + chunk_hdr_size()) + return -EINVAL; + + /* Test second contiguous chunk with unaligned size */ + addr2 = selftest_alloc(MIN_ALLOC_SIZE + 1); + if (!addr2) + return -EINVAL; + addr3 = selftest_alloc(0); + if (!addr3 || + addr3 != addr2 + (2 * MIN_ALLOC_SIZE) + chunk_hdr_size()) + return -EINVAL; + + selftest_free(addr3); + + /* Test chunk recycling */ + selftest_free(addr1); + if (addr1 != selftest_alloc(0)) + return -EINVAL; + + /* Test chunk forward merging */ + addr3 = selftest_alloc(0); + selftest_free(addr2); + selftest_free(addr1); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 2)) + return -EINVAL; + + selftest_free(addr1); + + /* Test chunk splitting */ + if (addr1 != selftest_alloc(0)) + return -EINVAL; + if (addr2 != selftest_alloc(0)) + return -EINVAL; + + /* Test chunk backward merging */ + selftest_free(addr1); + selftest_free(addr2); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 2)) + return -EINVAL; + + selftest_free(addr1); + + /* Test chunk 3-way merging */ + addr1 = selftest_alloc(0); + addr2 = selftest_alloc(0); + addr4 = selftest_alloc(0); + selftest_free(addr1); + selftest_free(addr3); + selftest_free(addr2); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 3)) + return -EINVAL; + + selftest_free(addr4); + selftest_free(addr1); + + /* Test reclaiming */ + if (addr1 != selftest_alloc(0)) + return -EINVAL; + if (addr2 != selftest_alloc(PAGE_SIZE * 2)) + return -EINVAL; + addr3 = selftest_alloc(0); + addr4 = selftest_alloc(PAGE_SIZE); + + /* Test reclaiming the last chunk of the list */ + selftest_free(addr4); + hyp_allocator_reclaim(allocator, &host_mc, SELFTEST_MAX_PAGES); + if (host_mc.nr_pages != SELFTEST_MAX_PAGES - 3) + return -EINVAL; + + /* Test punching a hole in the middle of a free chunk ... */ + selftest_free(addr2); + hyp_allocator_reclaim(allocator, &host_mc, SELFTEST_MAX_PAGES); + if (host_mc.nr_pages != SELFTEST_MAX_PAGES - 2) + return -EINVAL; + + if (selftest_alloc(PAGE_SIZE)) + return -EINVAL; + if (selftest_errno() != -ENOMEM) + return -EINVAL; + + /* ... and to refill this hole */ + ret = hyp_allocator_topup(allocator, &host_mc); + if (ret) + return ret; + /* Chunk at addr2 was made smaller by the reclaim */ + if (addr2 != selftest_alloc(PAGE_SIZE)) + return -EINVAL; + + /* Test reclaiming the entire allocator from the host */ + selftest_free(addr3); + selftest_free(addr2); + selftest_free(addr1); + if (addr1 != selftest_alloc(SELFTEST_MAX_PAGES * PAGE_SIZE - chunk_hdr_size())) + return -EINVAL; + selftest_free(addr1); + + return 0; +} +#else +static int selftest_init(void) { return 0; } +#endif
diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c index add9383..eba1436 100644 --- a/arch/arm64/kvm/hyp/nvhe/events.c +++ b/arch/arm64/kvm/hyp/nvhe/events.c
@@ -7,8 +7,16 @@ #include <nvhe/mm.h> #include <nvhe/trace.h> +#include <asm/kvm_hypevents.h> + #include <nvhe/define_events.h> +/* + * The hyp_printk event is not declared with HYP_EVENT in kvm_hypevents.h, + * so we manually add the boilerplate here. + */ +HYP_EVENT(hyp_printk, 0, 0, 0, 0); + int __tracing_enable_event(unsigned short id, bool enable) { struct hyp_event_id *event_id = &__hyp_event_ids_start[id];
diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index a327c2b..5ad74ab 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c
@@ -42,6 +42,9 @@ */ #define HOST_FFA_ID 0 +#define FFA_NOTIF_RECEIVER_ENDP_MASK GENMASK(15, 0) +#define FFA_NOTIF_SENDER_ENDP_MASK GENMASK(31, 16) + /* * A buffer to hold the maximum descriptor size we can see from the host, * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP @@ -71,6 +74,21 @@ static u32 hyp_ffa_version; static bool has_version_negotiated; static hyp_spinlock_t version_lock; +static bool ffa_check_unused_args_sbz(struct kvm_cpu_context *ctxt, int first_reg) +{ + DECLARE_REG(u32, func_id, ctxt, 0); + int reg, end_reg = 7; + + if (hyp_ffa_version >= FFA_VERSION_1_2 && ARM_SMCCC_IS_64(func_id)) + end_reg = 17; + for (reg = first_reg; reg <= end_reg; reg++) { + if (cpu_reg(ctxt, reg)) + return true; + } + + return false; +} + static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno) { *res = (struct arm_smccc_1_2_regs) { @@ -239,6 +257,11 @@ static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res, int ret = 0; void *rx_virt, *tx_virt; + if (ffa_check_unused_args_sbz(ctxt, 4)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) { ret = FFA_RET_INVALID_PARAMETERS; goto out; @@ -315,6 +338,11 @@ static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, id, ctxt, 1); int ret = 0; + if (ffa_check_unused_args_sbz(ctxt, 2)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (id != HOST_FFA_ID) { ret = FFA_RET_INVALID_PARAMETERS; goto out; @@ -421,6 +449,11 @@ static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, int ret = FFA_RET_INVALID_PARAMETERS; u32 nr_ranges; + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) goto out; @@ -483,6 +516,11 @@ static void __do_ffa_mem_xfer(const u64 func_id, int ret = 0; size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version); + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (addr_mbz || npages_mbz || fraglen > len || fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { ret = FFA_RET_INVALID_PARAMETERS; @@ -586,6 +624,11 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, int ret = 0; u64 handle; + if (ffa_check_unused_args_sbz(ctxt, 4)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + handle = PACK_HANDLE(handle_lo, handle_hi); hyp_spin_lock(&host_buffers.lock); @@ -691,14 +734,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_RXTX_MAP: case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: - /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_BITMAP_CREATE: - case FFA_NOTIFICATION_BITMAP_DESTROY: - case FFA_NOTIFICATION_BIND: - case FFA_NOTIFICATION_UNBIND: - case FFA_NOTIFICATION_SET: - case FFA_NOTIFICATION_GET: - case FFA_NOTIFICATION_INFO_GET: /* Optional interfaces added in FF-A 1.2 */ case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */ case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */ @@ -785,6 +820,11 @@ static void do_ffa_version(struct arm_smccc_1_2_regs *res, { DECLARE_REG(u32, ffa_req_version, ctxt, 1); + if (ffa_check_unused_args_sbz(ctxt, 2)) { + res->a0 = FFA_RET_NOT_SUPPORTED; + return; + } + if (FFA_MAJOR_VERSION(ffa_req_version) != 1) { res->a0 = FFA_RET_NOT_SUPPORTED; return; @@ -834,6 +874,11 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, flags, ctxt, 5); u32 count, partition_sz, copy_sz; + if (ffa_check_unused_args_sbz(ctxt, 6)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + hyp_spin_lock(&host_buffers.lock); if (!host_buffers.rx) { ffa_to_smccc_res(res, FFA_RET_BUSY); @@ -878,6 +923,140 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, hyp_spin_unlock(&host_buffers.lock); } +static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, func_id, ctxt, 0); + DECLARE_REG(u32, vmid, ctxt, 1); + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, func_id == FFA_NOTIFICATION_BITMAP_CREATE ? 3 : 2)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (vmid != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags > 1) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, reserved, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 5) || reserved) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_SENDER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags & GENMASK(15, 2)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (ffa_check_unused_args_sbz(ctxt, 3)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags & GENMASK(31, 4)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_info_get(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 1)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -906,6 +1085,11 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) switch (func_id) { case FFA_FEATURES: + if (ffa_check_unused_args_sbz(host_ctxt, 3)) { + ffa_to_smccc_res(&res, FFA_RET_INVALID_PARAMETERS); + goto out_handled; + } + if (!do_ffa_features(&res, host_ctxt)) return false; goto out_handled; @@ -936,6 +1120,26 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_PARTITION_INFO_GET: do_ffa_part_get(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_BITMAP_CREATE: + case FFA_NOTIFICATION_BITMAP_DESTROY: + do_ffa_notif_bitmap(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_BIND: + do_ffa_notif_bind(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_UNBIND: + do_ffa_notif_unbind(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_SET: + do_ffa_notif_set(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_GET: + do_ffa_notif_get(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_INFO_GET: + case FFA_FN64_NOTIFICATION_INFO_GET: + do_ffa_notif_info_get(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id))
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index ac64a03..137a814 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -4,10 +4,13 @@ * Author: Andrew Scull <ascull@google.com> */ +#include <kvm/arm_hypercalls.h> + #include <hyp/adjust_pc.h> #include <hyp/switch.h> #include <linux/irqchip/arm-gic-v3.h> +#include <uapi/linux/psci.h> #include <asm/pgtable-types.h> #include <asm/kvm_asm.h> @@ -17,6 +20,7 @@ #include <asm/kvm_hypevents.h> #include <asm/kvm_mmu.h> +#include <nvhe/alloc.h> #include <nvhe/ffa.h> #include <nvhe/mem_protect.h> #include <nvhe/mm.h> @@ -26,11 +30,449 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* + * Define a hypercall handler: handle_<name> unmarshals the arguments from + * the host context and hands them, correctly typed, to the body that + * follows the macro. The parameter list is type-checked against the + * signature declared in <asm/kvm_hcall.h>, so the handler cannot drift + * from what the typed caller stubs marshal in. Modelled on the syscall + * wrappers. + */ +/* Truncate the fixed list of argument registers to the declared signature. */ +#define KVM_HOST_HCALL_REGS(...) \ + __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), __KVM_HCALL_ARGS \ + ,, cpu_reg(host_ctxt, 1),, cpu_reg(host_ctxt, 2) \ + ,, cpu_reg(host_ctxt, 3),, cpu_reg(host_ctxt, 4) \ + ,, cpu_reg(host_ctxt, 5),, cpu_reg(host_ctxt, 6)) + +#define set_cpu_reg_ulong(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_u64(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_int(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_void(ctxt, r, v) { v; } +#define set_cpu_reg(ctxt, r, t, v) set_cpu_reg_##t(ctxt, r, v) + +#define DEFINE_KVM_HOST_HCALL(ret, name, ...) \ + static kvm_host_hcall_sig_##name __do_##name; \ + static __always_inline \ + ret __se_##name(__KVM_HCALL_MAP(__KVM_HCALL_LONG, __VA_ARGS__)) \ + { \ + return __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_CAST, __VA_ARGS__)); \ + } \ + static void handle_##name(struct kvm_cpu_context *host_ctxt) \ + { \ + set_cpu_reg(host_ctxt, 1, ret, __se_##name(KVM_HOST_HCALL_REGS(__VA_ARGS__))); \ + } \ + static ret __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) + +#define DEFINE_KVM_HOST_HCALL0(ret, name) \ + static kvm_host_hcall_sig_##name __do_##name; \ + static void handle_##name(struct kvm_cpu_context *host_ctxt) \ + { \ + set_cpu_reg(host_ctxt, 1, ret, __do_##name()); \ + } \ + static ret __do_##name(void) + +/* + * Encode a hypervisor request in the host SMCCC return registers for known + * error numbers. + * + * Must be paired with pkvm_call_hyp_req() on the host side. + */ +static int errno_to_smccc(int ret) +{ + struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; + + switch (ret) { + case -ENOMEM: { + u32 nr_pages = hyp_alloc_topup_needed(); + + if (nr_pages) { + req.type = PKVM_HYP_REQ_HYP_ALLOC; + req.mem.nr_pages = nr_pages; + } + break; + } + } + + pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); + + return ret; +} + /* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ unsigned int hyp_gicv3_nr_lr; void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); +typedef void (*hyp_entry_exit_handler_fn)(struct pkvm_hyp_vcpu *); + +static bool pvm_sys64_is_write(u64 esr) +{ + return (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_WRITE; +} + +static void handle_pvm_entry_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + /* Exceptions have priority; the host injects none on WFx. */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) + return; + + if (vcpu_get_flag(host_vcpu, INCREMENT_PC)) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } +} + +static void handle_pvm_entry_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + bool pc_update; + + /* Exceptions have priority over anything else */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { + /* A host-requested exception on SYS64 is always an UNDEF. */ + u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); + return; + } + + /* Handle PC increment on a host-emulated access */ + pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC); + if (pc_update) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } + + /* If the host emulated a read access, update the register */ + if (pc_update && !pvm_sys64_is_write(kvm_vcpu_get_esr(vcpu))) { + /* r0 as transfer register between the guest and the host. */ + u64 rt_val = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + int rt = kvm_vcpu_sys_get_rt(vcpu); + + vcpu_set_reg(vcpu, rt, rt_val); + } +} + +static void handle_pvm_entry_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + unsigned long cpsr = *vcpu_cpsr(vcpu); + u32 esr = ESR_ELx_IL; + + if (!vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) + return; + + /* The host's only IABT injection: an external abort. */ + if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) + esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT); + else + esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); + + esr |= ESR_ELx_FSC_EXTABT; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); + + /* Injected by __kvm_adjust_pc() on entry. */ + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); +} + +/* + * Clamp MMIO data to the access width, so a write does not leak the + * register's upper bits and a read takes no bits beyond the load. The + * host applies endianness. + */ +static inline u64 kvm_mmio_clamp_data(struct kvm_vcpu *vcpu, u64 val) +{ + unsigned int len = kvm_vcpu_dabt_get_as(vcpu); + + return val & GENMASK_U64(len * 8 - 1, 0); +} + +/* + * Complete an MMIO load: sign-extend from EL2's own syndrome, as the + * architecture does. + */ +static inline u64 kvm_mmio_read_data(struct kvm_vcpu *vcpu, u64 val) +{ + val = kvm_mmio_clamp_data(vcpu, val); + + if (kvm_vcpu_dabt_issext(vcpu)) + val = sign_extend64(val, kvm_vcpu_dabt_get_as(vcpu) * 8 - 1); + + if (!kvm_vcpu_dabt_issf(vcpu)) + val &= GENMASK_U64(31, 0); + + return val; +} + +static void handle_pvm_entry_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + bool pc_update; + + /* Exceptions have priority over anything else */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { + unsigned long cpsr = *vcpu_cpsr(vcpu); + u32 esr = ESR_ELx_IL; + + if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) + esr |= (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT); + else + esr |= (ESR_ELx_EC_DABT_CUR << ESR_ELx_EC_SHIFT); + + esr |= ESR_ELx_FSC_EXTABT; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); + + /* Injected by __kvm_adjust_pc() on entry. */ + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); + + /* Cancel any in-flight MMIO */ + vcpu->mmio_needed = false; + return; + } + + /* Handle PC increment on MMIO, or on a CMO the host skipped */ + pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC) && + (vcpu->mmio_needed || esr_dabt_is_cm(kvm_vcpu_get_esr(vcpu))); + if (pc_update) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } + + /* If the host emulated an MMIO read, update the register */ + if (pc_update && vcpu->mmio_needed && !kvm_vcpu_dabt_iswrite(vcpu)) { + /* r0 as transfer register between the guest and the host. */ + u64 rd_val = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + int rd = kvm_vcpu_dabt_get_rd(vcpu); + + rd_val = kvm_mmio_read_data(vcpu, rd_val); + vcpu_set_reg(vcpu, rd, rd_val); + } + + vcpu->mmio_needed = false; +} + +static void handle_pvm_entry_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 ret = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + u32 psci_fn = smccc_get_function(vcpu); + + switch (psci_fn) { + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + /* + * Roll back a CPU_ON the host failed, unless the target + * already reached ON: it is running, and the guest sees + * SUCCESS. + */ + if (ret != PSCI_RET_SUCCESS) { + unsigned long cpu_id = smccc_get_arg1(vcpu); + struct pkvm_hyp_vcpu *target_vcpu; + struct pkvm_hyp_vm *hyp_vm; + int prev; + + hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + target_vcpu = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); + + /* + * pvm_psci_vcpu_on() resolved this MPIDR and vcpus[] + * entries are never removed, so the lookup cannot miss. + * The release orders this vCPU's reset_state writes + * before OFF, for the next CPU_ON. + */ + prev = cmpxchg_release(&target_vcpu->power_state, + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, + PSCI_0_2_AFFINITY_LEVEL_OFF); + switch (prev) { + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + /* + * Leave reset_state.reset set: a clear races a + * fresh CPU_ON's publish. The stale pc/r0/be are + * the guest's own. ALREADY_ON is PSCI's retry + * signal for a CPU_ON that raced the CPU_OFF. + */ + if (ret != PSCI_RET_ALREADY_ON) + ret = PSCI_RET_INTERNAL_FAILURE; + break; + case PSCI_0_2_AFFINITY_LEVEL_ON: + case PSCI_0_2_AFFINITY_LEVEL_OFF: + /* Target already ran (and may have stopped). */ + ret = PSCI_RET_SUCCESS; + break; + default: + ret = PSCI_RET_INTERNAL_FAILURE; + break; + } + } + + break; + default: + break; + } + + vcpu_set_reg(vcpu, 0, ret); +} + +/* The host's view of a syndrome: the guest register index is withheld. */ +static u64 pvm_host_esr(u64 esr) +{ + switch (ESR_ELx_EC(esr)) { + case ESR_ELx_EC_WFx: + return esr & ~ESR_ELx_WFx_ISS_RN; + case ESR_ELx_EC_SYS64: + return esr & ~ESR_ELx_SYS64_ISS_RT_MASK; + case ESR_ELx_EC_DABT_LOW: + return esr & ~ESR_ELx_SRT_MASK; + default: + return esr; + } +} + +/* + * The host's view of PSTATE: the mode, with SErrors masked so that the host + * pends an SError through HCR_EL2.VSE rather than emulating the entry. + */ +static unsigned long pvm_host_pstate(unsigned long pstate) +{ + return (pstate & PSR_MODE_MASK) | PSR_A_BIT; +} + +static void handle_pvm_exit_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); +} + +static void handle_pvm_exit_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 esr = kvm_vcpu_get_esr(vcpu); + + /* The mode is required for the host to emulate some sysregs */ + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); + + /* r0 as transfer register between the guest and the host. */ + if (pvm_sys64_is_write(esr)) { + int rt = kvm_vcpu_sys_get_rt(vcpu); + u64 rt_val = vcpu_get_reg(vcpu, rt); + + vcpu_set_reg(host_vcpu, 0, rt_val); + } +} + +static void handle_pvm_exit_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; +} + +static void handle_pvm_exit_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 sctlr; + + /* + * EL2 has no memslot view: a decodable data abort is prepared as MMIO + * for the host to resolve. On unbacked memory the host injects an SEA + * for one with ISV clear (LDP/STP, atomics), which EL2 does not + * decode, and skips a cache maintenance operation, as for any guest. + */ + vcpu->mmio_needed = kvm_vcpu_dabt_isvalid(vcpu); + + /* r0 as transfer register between the guest and the host. */ + if (vcpu->mmio_needed && kvm_vcpu_dabt_iswrite(vcpu)) { + int rt = kvm_vcpu_dabt_get_rd(vcpu); + u64 rt_val = vcpu_get_reg(vcpu, rt); + + rt_val = kvm_mmio_clamp_data(vcpu, rt_val); + vcpu_set_reg(host_vcpu, 0, rt_val); + } + + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); + host_vcpu->arch.fault.far_el2 = vcpu->arch.fault.far_el2 & GENMASK(11, 0); + host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; + sctlr = __vcpu_sys_reg(vcpu, SCTLR_EL1) & (SCTLR_ELx_EE | SCTLR_EL1_E0E); + __vcpu_assign_sys_reg(host_vcpu, SCTLR_EL1, sctlr); +} + +static void handle_pvm_exit_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + int n, i; + + switch (smccc_get_function(vcpu)) { + /* + * CPU_ON: the host uses only the target MPIDR (x1). EL2 resets the + * target from its own copy of the entry point and context id. + */ + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + n = 2; + break; + + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + n = 1; + break; + + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + n = 3; + break; + + /* Unreachable: kvm_handle_pvm_hvc64() forwards only the calls above. */ + default: + hyp_panic(); + } + + /* Pass the HVC function id (r0) and its arguments. */ + for (i = 0; i < n; i++) + vcpu_set_reg(host_vcpu, i, vcpu_get_reg(vcpu, i)); +} + +static const hyp_entry_exit_handler_fn entry_hyp_pvm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_WFx] = handle_pvm_entry_wfx, + [ESR_ELx_EC_SYS64] = handle_pvm_entry_sys64, + [ESR_ELx_EC_IABT_LOW] = handle_pvm_entry_iabt, + [ESR_ELx_EC_DABT_LOW] = handle_pvm_entry_dabt, + [ESR_ELx_EC_HVC64] = handle_pvm_entry_hvc64, +}; + +static const hyp_entry_exit_handler_fn exit_hyp_pvm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_WFx] = handle_pvm_exit_wfx, + [ESR_ELx_EC_SYS64] = handle_pvm_exit_sys64, + [ESR_ELx_EC_IABT_LOW] = handle_pvm_exit_iabt, + [ESR_ELx_EC_DABT_LOW] = handle_pvm_exit_dabt, + [ESR_ELx_EC_HVC64] = handle_pvm_exit_hvc64, +}; + static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) { __vcpu_assign_sys_reg(vcpu, ZCR_EL1, read_sysreg_el1(SYS_ZCR)); @@ -143,6 +585,35 @@ static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; } +static void flush_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + /* A hyp vcpu has no offset, and sees vtime == ptime. */ + write_sysreg(0, cntvoff_el2); + write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CVAL_EL0), SYS_CNTV_CVAL); + isb(); + write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CTL_EL0), SYS_CNTV_CTL); +} + +static void sync_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + /* + * Preserve the vtimer state so that it is always correct, + * even if the host tries to make a mess. + */ + __vcpu_assign_sys_reg(vcpu, CNTV_CVAL_EL0, read_sysreg_el0(SYS_CNTV_CVAL)); + __vcpu_assign_sys_reg(vcpu, CNTV_CTL_EL0, read_sysreg_el0(SYS_CNTV_CTL)); +} + static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu, struct kvm_vcpu *to_vcpu) { @@ -189,6 +660,9 @@ static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) { @@ -207,6 +681,9 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) host_vcpu->arch.vcpu_debug_state = hyp_vcpu->vcpu.arch.vcpu_debug_state; else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) @@ -217,6 +694,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; u64 host_hcr_mask = PKVM_HCR_EL2_HOST_PVM; + hyp_entry_exit_handler_fn ec_handler; + u8 esr_ec; fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); @@ -229,42 +708,75 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); + + hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; + hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; host_hcr_mask = PKVM_HCR_EL2_HOST_NPVM; - } else { - hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; } /* __hyp_running_vcpu must be NULL in a guest context. */ hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; - hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; /* - * HCR_EL2.VSE is host-owned (a pending virtual SError to inject), not a - * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE - * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. + * A host-injected vSError is masked by the guest's own PSTATE.A, so it + * applies to protected guests too. */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~host_hcr_mask; hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & host_hcr_mask; - - hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; - - hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; + hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; flush_hyp_vgic_state(hyp_vcpu); + flush_hyp_timer_state(hyp_vcpu); hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; + + switch (ARM_EXCEPTION_CODE(hyp_vcpu->exit_code)) { + case ARM_EXCEPTION_IRQ: + case ARM_EXCEPTION_EL1_SERROR: + case ARM_EXCEPTION_IL: + break; + case ARM_EXCEPTION_TRAP: + /* Nothing was marshalled for this trap, see sync_hyp_vcpu(). */ + if (ARM_SERROR_PENDING(hyp_vcpu->exit_code)) + break; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = entry_hyp_pvm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + } + break; + default: + BUG(); + } + + hyp_vcpu->exit_code = 0; } -static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) +static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + hyp_entry_exit_handler_fn ec_handler; + u8 esr_ec; fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { - host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; + /* + * Protected: the host sees ESR_EL2 as EL2 took it, register + * index withheld; the fault addresses stay withheld unless the + * EC handler below adds them. + */ + host_vcpu->arch.fault = (struct kvm_vcpu_fault_info) { + .esr_el2 = pvm_host_esr(hyp_vcpu->vcpu.arch.fault.esr_el2), + .disr_el1 = hyp_vcpu->vcpu.arch.fault.disr_el1, + }; } else { + /* Non-protected: the host gets the full fault. */ + host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; + host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; /* * PC feeds trace_kvm_exit(), PSTATE.SS the host software-step * machine, and both run before the next on-demand ctxt sync. @@ -273,18 +785,45 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate; } - host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; + switch (ARM_EXCEPTION_CODE(exit_reason)) { + case ARM_EXCEPTION_IRQ: + break; + case ARM_EXCEPTION_TRAP: + /* SError pending: not handled at EL2, the guest replays it. */ + if (ARM_SERROR_PENDING(exit_reason)) + break; - host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; + /* Per-EC marshalling is for protected guests only. */ + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = exit_hyp_pvm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + } + break; + case ARM_EXCEPTION_EL1_SERROR: + case ARM_EXCEPTION_IL: + break; + default: + BUG(); + } + + /* Cleared by hardware once the guest takes the vSError. */ + host_vcpu->arch.hcr_el2 &= ~HCR_VSE; + host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; sync_hyp_vgic_state(hyp_vcpu); + sync_hyp_timer_state(hyp_vcpu); + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + vcpu_clear_flag(host_vcpu, PC_UPDATE_REQ); + + hyp_vcpu->exit_code = exit_reason; } -static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, + pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(unsigned int, vcpu_idx, host_ctxt, 2); - DECLARE_REG(u64, hcr_el2, host_ctxt, 3); struct pkvm_hyp_vcpu *hyp_vcpu; hyp_vcpu = pkvm_load_hyp_vcpu(handle, vcpu_idx); @@ -292,16 +831,24 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) return; if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + u64 dfr0 = read_sysreg(id_aa64dfr0_el1); + /* Propagate WFx trapping flags */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); + + /* HPMN == 0 is reserved without FEAT_HPMN0. */ + if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0))) + u64p_replace_bits(&hyp_vcpu->vcpu.arch.mdcr_el2, + FIELD_GET(ARMV8_PMU_PMCR_N, read_sysreg(pmcr_el0)), + MDCR_EL2_HPMN); } else { memcpy(&hyp_vcpu->vcpu.arch.fgt, hyp_vcpu->host_vcpu->arch.fgt, sizeof(hyp_vcpu->vcpu.arch.fgt)); } } -static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); @@ -317,7 +864,7 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) } } -static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -328,10 +875,9 @@ static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) sync_hyp_vcpu_state(hyp_vcpu); } -static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, +static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *host_vcpu, struct pkvm_hyp_vcpu **hyp_vcpup) { - struct kvm_vcpu *host_vcpu = kern_hyp_va(arg); struct pkvm_hyp_vcpu *hyp_vcpu = NULL; if (unlikely(is_protected_kvm_enabled())) { @@ -347,34 +893,26 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, return host_vcpu; } -#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup) \ - ({ \ - DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr); \ - __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ - }) +static struct kvm_vcpu * +__get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if __kern *cpu_if, + struct pkvm_hyp_vcpu **hyp_vcpup) +{ + struct vgic_v3_cpu_if *host_cpu_if = kern_hyp_va_host(cpu_if); + struct kvm_vcpu *host_vcpu = container_of(host_cpu_if, struct kvm_vcpu, + arch.vgic_cpu.vgic_v3); -#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup) \ - ({ \ - DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\ - struct kvm_vcpu *__vcpu = container_of(cif, \ - struct kvm_vcpu, \ - arch.vgic_cpu.vgic_v3); \ - \ - __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ - }) + return __get_host_hyp_vcpus(host_vcpu, hyp_vcpup); +} -static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __kvm_vcpu_run, struct kvm_vcpu __kern *, vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - int ret; + int ret = ARM_EXCEPTION_IL; - host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu); - - if (!host_vcpu) { - ret = -EINVAL; - goto out; - } + host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); + if (!host_vcpu) + return ret; if (unlikely(hyp_vcpu)) { /* @@ -383,213 +921,245 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) * loading a vcpu. Therefore, if SME features enabled the host * is misbehaving. */ - if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) { - ret = -EINVAL; - goto out; + if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) + return ret; + + /* + * ON has a single writer, pkvm_reset_vcpu() on this CPU, so + * READ_ONCE suffices. ON_PENDING takes the reset; -ECANCELED + * is a rollback that raced it. + */ + switch (READ_ONCE(hyp_vcpu->power_state)) { + case PSCI_0_2_AFFINITY_LEVEL_ON: + break; + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + if (pkvm_reset_vcpu(hyp_vcpu)) + return ret; + break; + default: + return ret; } flush_hyp_vcpu(hyp_vcpu); ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); - sync_hyp_vcpu(hyp_vcpu); + sync_hyp_vcpu(hyp_vcpu, ret); } else { /* The host is fully trusted, run its vCPU directly. */ fpsimd_lazy_switch_to_guest(host_vcpu); ret = __kvm_vcpu_run(host_vcpu); fpsimd_lazy_switch_to_host(host_vcpu); } -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; - return refill_memcache(&hyp_vcpu->vcpu.arch.pkvm_memcache, - host_vcpu->arch.pkvm_memcache.nr_pages, - &host_vcpu->arch.pkvm_memcache); + return refill_memcache(&hyp_vcpu->vcpu.arch.stage2_mc, + host_vcpu->arch.stage2_mc.nr_pages, + &host_vcpu->arch.stage2_mc); } -static void handle___pkvm_host_donate_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, + u64, pfn, u64, gfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; + int ret; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || !pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; ret = pkvm_refill_memcache(hyp_vcpu); if (ret) - goto out; + return ret; - ret = __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); } -static void handle___pkvm_host_share_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, + u64, pfn, u64, gfn, u64, nr_pages, u64, prot) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; + int ret; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; ret = pkvm_refill_memcache(hyp_vcpu); if (ret) - goto out; + return ret; - ret = __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); } -static void handle___pkvm_host_unshare_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_unshare_guest(gfn, nr_pages, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_relax_perms_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, + u64, gfn, u64, prot) { - DECLARE_REG(u64, gfn, host_ctxt, 1); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 2); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; - ret = __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); } -static void handle___pkvm_host_wrprotect_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_wrprotect_guest(gfn, nr_pages, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_test_clear_young_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); - DECLARE_REG(bool, mkold, host_ctxt, 4); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_test_clear_young_guest(gfn, nr_pages, mkold, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, + u64, gfn) { - DECLARE_REG(u64, gfn, host_ctxt, 1); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; - ret = __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); } -static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) +/* + * PKVM_HOST_STATE_DIRTY names the authoritative copy, the host's when set. + * A loaded protected vCPU takes the request at its next entry instead. + */ +struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) { - DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; - __kvm_adjust_pc(kern_hyp_va(vcpu)); + if (!is_protected_kvm_enabled()) + return vcpu; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || vcpu == &hyp_vcpu->vcpu) + return vcpu; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return NULL; + + if (vcpu_get_flag(vcpu, PKVM_HOST_STATE_DIRTY)) + return vcpu; + + vcpu_copy_flag(&hyp_vcpu->vcpu, vcpu, PC_UPDATE_REQ); + return &hyp_vcpu->vcpu; } -static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt) +/* Reflect the consumed request back, otherwise it stays pending. */ +void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target) +{ + if (target != vcpu) + vcpu_copy_flag(vcpu, target, PC_UPDATE_REQ); +} + +DEFINE_KVM_HOST_HCALL(void, __kvm_adjust_pc, + struct kvm_vcpu __kern *, vcpu) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; + + host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); + if (host_vcpu) { + __kvm_adjust_pc(host_vcpu); + return; + } + + /* + * With no hyp vCPU loaded for it, the host vCPU may be unpinned, + * and so unmapped at EL2: its first run pins it. A pin fails only + * for memory the host isn't sharing, a bad pointer, so the request + * is dropped. + */ + host_vcpu = kern_hyp_va_host(vcpu); + if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) + return; + + __kvm_adjust_pc(host_vcpu); + hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); +} + +DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) { __kvm_flush_vm_context(); } -static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); - DECLARE_REG(int, level, host_ctxt, 3); - - __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); + __kvm_tlb_flush_vmid_ipa(kern_hyp_va_host(mmu), ipa, level); } -static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); - DECLARE_REG(int, level, host_ctxt, 3); - - __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level); + __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va_host(mmu), ipa, level); } -static void -handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, start, host_ctxt, 2); - DECLARE_REG(unsigned long, pages, host_ctxt, 3); - - __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages); + __kvm_tlb_flush_vmid_range(kern_hyp_va_host(mmu), start, pages); } -static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, + struct kvm_s2_mmu __kern *, mmu) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - - __kvm_tlb_flush_vmid(kern_hyp_va(mmu)); + __kvm_tlb_flush_vmid(kern_hyp_va_host(mmu)); } -static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); struct pkvm_hyp_vm *hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) @@ -599,19 +1169,19 @@ static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) put_pkvm_hyp_vm(hyp_vm); } -static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, + struct kvm_s2_mmu __kern *, mmu) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - - __kvm_flush_cpu_context(kern_hyp_va(mmu)); + __kvm_flush_cpu_context(kern_hyp_va_host(mmu)); } -static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, + u64, cntvoff) { - __kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1)); + __kvm_timer_set_cntvoff(cntvoff); } -static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) { u64 tmp; @@ -620,23 +1190,23 @@ static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) write_sysreg_el2(tmp, SYS_SCTLR); } -static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) { - cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config(); + return __vgic_v3_get_gic_config(); } -static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) { __vgic_v3_init_lrs(); } -static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, - &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); if (!host_vcpu) return; @@ -658,13 +1228,13 @@ static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) } } -static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, - &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); if (!host_vcpu) return; @@ -691,48 +1261,39 @@ static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt } } -static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init, + phys_addr_t, phys, unsigned long, size, + unsigned long *, per_cpu_base, u32, hyp_va_bits) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - DECLARE_REG(unsigned long, size, host_ctxt, 2); - DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 3); - DECLARE_REG(u32, hyp_va_bits, host_ctxt, 4); - /* * __pkvm_init() will return only if an error occurred, otherwise it * will tail-call in __pkvm_init_finalise() which will have to deal * with the host context directly. */ - cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); + return __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); } -static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, + enum arm64_hyp_spectre_vector, slot) { - DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot); + return pkvm_cpu_set_vector(slot); } -static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, + u64, pfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn); + return __pkvm_host_share_hyp(pfn); } -static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, + u64, pfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn); + return __pkvm_host_unshare_hyp(pfn); } -static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, + phys_addr_t, phys, size_t, size, u64, prot) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - DECLARE_REG(size_t, size, host_ctxt, 2); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3); - /* * __pkvm_create_private_mapping() populates a pointer with the * hypervisor start address of the allocation. @@ -743,161 +1304,237 @@ static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ct * Instead pass the allocation address as the return value (or return * ERR_PTR() on failure). */ - unsigned long haddr; + ulong haddr; int err = __pkvm_create_private_mapping(phys, size, prot, &haddr); if (err) - haddr = (unsigned long)ERR_PTR(err); + haddr = (ulong)ERR_PTR(err); - cpu_reg(host_ctxt, 1) = haddr; + return haddr; } -static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) { - cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); + return __pkvm_prot_finalize(); } -static void handle___pkvm_reserve_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) { - cpu_reg(host_ctxt, 1) = __pkvm_reserve_vm(); + return __pkvm_reserve_vm(); } -static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - __pkvm_unreserve_vm(handle); } -static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, + struct kvm __kern *, host_kvm, void __kern *, pgd_hva) { - DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); - DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2); - DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3); - - host_kvm = kern_hyp_va(host_kvm); - cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva); + return errno_to_smccc(__pkvm_init_vm(kern_hyp_va_host(host_kvm), pgd_hva)); } -static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2); - DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3); - - host_vcpu = kern_hyp_va(host_vcpu); - cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva); + return errno_to_smccc(__pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu))); } -static void handle___pkvm_vcpu_in_poison_fault(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) { - int ret; struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - ret = hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; - cpu_reg(host_ctxt, 1) = ret; + return hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; } -static void handle___pkvm_force_reclaim_guest_page(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, + phys_addr_t, phys) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_force_reclaim_page_guest(phys); + return __pkvm_host_force_reclaim_page_guest(phys); } -static void handle___pkvm_reclaim_dying_guest_page(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, + pkvm_handle_t, handle, u64, gfn) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_page(handle, gfn); + return __pkvm_reclaim_dying_guest_page(handle, gfn); } -static void handle___pkvm_start_teardown_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_start_teardown_vm(handle); + return __pkvm_start_teardown_vm(handle); } -static void handle___pkvm_finalize_teardown_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_finalize_teardown_vm(handle); + return __pkvm_finalize_teardown_vm(handle); } -static void handle___tracing_load(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) { - DECLARE_REG(unsigned long, desc_hva, host_ctxt, 1); - DECLARE_REG(size_t, desc_size, host_ctxt, 2); + struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; + int ret = -EPERM; - cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size); +#ifdef CONFIG_NVHE_EL2_DEBUG + ret = hyp_allocator_selftest(); + if (ret == -ENOMEM) { + req.type = PKVM_HYP_REQ_HYP_ALLOC_SELFTEST; + req.mem.nr_pages = hyp_alloc_selftest_topup_needed(); + } +#endif + pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); + + return ret; } -static void handle___tracing_unload(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, + enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) +{ + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); + struct kvm_hyp_memcache host_mc = { + .head = head, + .nr_pages = nr_pages, + }; + int ret; + + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + ret = hyp_alloc_topup(&host_mc); + break; + case PKVM_TOPUP_HYP_ALLOC_SELFTEST: + ret = hyp_alloc_selftest_topup(&host_mc); + break; + default: + ret = -EINVAL; + } + + cpu_reg(host_ctxt, 2) = host_mc.head; + cpu_reg(host_ctxt, 3) = host_mc.nr_pages; + + return ret; +} + +DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, + enum pkvm_topup_id, id, unsigned long, target) +{ + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); + struct kvm_hyp_memcache host_mc = {}; + int ret = 0; + + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + hyp_alloc_reclaim(&host_mc, target); + break; + case PKVM_TOPUP_HYP_ALLOC_SELFTEST: + hyp_alloc_selftest_reclaim(&host_mc, target); + break; + default: + ret = -EINVAL; + } + + cpu_reg(host_ctxt, 2) = host_mc.head; + cpu_reg(host_ctxt, 3) = host_mc.nr_pages; + + return ret; +} + +DEFINE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, + enum pkvm_topup_id, id) +{ + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + return hyp_alloc_reclaimable(); + default: + return 0; + } +} + +DEFINE_KVM_HOST_HCALL(int, __tracing_load, + void __kern *, desc_hva, size_t, desc_size) +{ + return errno_to_smccc(__tracing_load(desc_hva, desc_size)); +} + +DEFINE_KVM_HOST_HCALL0(void, __tracing_unload) { __tracing_unload(); } -static void handle___tracing_enable(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_enable, + bool, enable) { - DECLARE_REG(bool, enable, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_enable(enable); + return __tracing_enable(enable); } -static void handle___tracing_swap_reader(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_swap_reader, + unsigned int, cpu) { - DECLARE_REG(unsigned int, cpu, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_swap_reader(cpu); + return __tracing_swap_reader(cpu); } -static void handle___tracing_update_clock(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __tracing_update_clock, + u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) { - DECLARE_REG(u32, mult, host_ctxt, 1); - DECLARE_REG(u32, shift, host_ctxt, 2); - DECLARE_REG(u64, epoch_ns, host_ctxt, 3); - DECLARE_REG(u64, epoch_cyc, host_ctxt, 4); - __tracing_update_clock(mult, shift, epoch_ns, epoch_cyc); } -static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_reset, + unsigned int, cpu) { - DECLARE_REG(unsigned int, cpu, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_reset(cpu); + return __tracing_reset(cpu); } -static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_enable_event, + unsigned short, id, bool, enable) { - DECLARE_REG(unsigned short, id, host_ctxt, 1); - DECLARE_REG(bool, enable, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable); + return __tracing_enable_event(id, enable); } -static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __tracing_write_event, + u64, id) { - DECLARE_REG(u64, id, host_ctxt, 1); - trace_selftest(id); } -static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_resident, + struct vgic_v5_cpu_if __kern *, cpu_if) { - DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); + if (unlikely(is_protected_kvm_enabled())) + return; - __vgic_v5_save_apr(kern_hyp_va(cpu_if)); + __vgic_v5_make_resident(kern_hyp_va_host(cpu_if)); } -static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident, + struct vgic_v5_cpu_if __kern *, cpu_if) { - DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); + if (unlikely(is_protected_kvm_enabled())) + return; - __vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if)); + __vgic_v5_make_non_resident(kern_hyp_va_host(cpu_if)); +} + +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +{ + __vgic_v5_save_apr(kern_hyp_va_host(cpu_if)); +} + +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +{ + __vgic_v5_restore_vmcr_apr(kern_hyp_va_host(cpu_if)); +} + +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_vdpend, + u32, intid, bool, pending, u16, vm) +{ + if (unlikely(is_protected_kvm_enabled())) + return; + + __vgic_v5_vdpend(intid, pending, vm); } typedef void (*hcall_t)(struct kvm_cpu_context *); @@ -912,6 +1549,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__kvm_enable_ssbs), HANDLE_FUNC(__vgic_v3_init_lrs), HANDLE_FUNC(__vgic_v3_get_gic_config), + HANDLE_FUNC(__pkvm_hyp_alloc_selftest), HANDLE_FUNC(__pkvm_prot_finalize), HANDLE_FUNC(__kvm_adjust_pc), @@ -933,11 +1571,18 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__tracing_write_event), HANDLE_FUNC(__vgic_v3_save_aprs), HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), + HANDLE_FUNC(__vgic_v5_make_resident), + HANDLE_FUNC(__vgic_v5_make_non_resident), + HANDLE_FUNC(__vgic_v5_vdpend), HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), + HANDLE_FUNC(__pkvm_hyp_topup), + HANDLE_FUNC(__pkvm_hyp_reclaim), + HANDLE_FUNC(__pkvm_hyp_reclaimable), HANDLE_FUNC(__pkvm_host_share_hyp), HANDLE_FUNC(__pkvm_host_unshare_hyp), + HANDLE_FUNC(__pkvm_host_donate_guest), HANDLE_FUNC(__pkvm_host_share_guest), HANDLE_FUNC(__pkvm_host_unshare_guest),
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S index 7a02837..8922242 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -17,7 +17,6 @@ HYP_SECTION(.data..ro_after_init) HYP_SECTION(.rodata) #ifdef CONFIG_NVHE_EL2_TRACING - . = ALIGN(PAGE_SIZE); BEGIN_HYP_SECTION(.event_ids) *(SORT(.hyp.event_ids.*)) END_HYP_SECTION
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index a6a47c1..c3035aa 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -540,7 +540,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) int ret; hyp_assert_lock_held(&host_mmu.lock); - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level, 0); if (ret) return ret; @@ -930,7 +930,7 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep, s8 level; int ret; - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) return ret; if (guest_pte_is_poisoned(pte)) @@ -979,7 +979,7 @@ int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu) ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu)); guest_lock_component(vm); - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) goto unlock; @@ -1043,7 +1043,7 @@ int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) ret = 0; WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_SHARED_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); WARN_ON(__host_set_page_state_range(phys, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED)); unlock: guest_unlock_component(vm); @@ -1078,7 +1078,7 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) PKVM_ID_GUEST, meta)); WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); host_unlock_component(); @@ -1335,7 +1335,7 @@ static int host_stage2_get_guest_info(phys_addr_t phys, struct pkvm_hyp_vm **vm, return -EPERM; } - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level, 0); if (ret) return ret; @@ -1442,7 +1442,7 @@ static int __guest_check_pgtable_memcache(struct pkvm_hyp_vcpu *vcpu) { struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); - if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) + if (vcpu->vcpu.arch.stage2_mc.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) return -ENOMEM; return 0; @@ -1476,7 +1476,7 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu) PKVM_ID_GUEST, meta)); WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); @@ -1547,7 +1547,7 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, pkvm_mkstate(prot, PKVM_PAGE_SHARED_BORROWED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); @@ -1564,7 +1564,7 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip s8 level; int ret; - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) return ret; if (!kvm_pte_valid(pte))
diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 29ab5ee..b95d98c 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -91,6 +91,59 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) return ret; } +/** + * pkvm_map_private_va_range() - Map a physical range into the private VA range + * @haddr: The virtual address in the private range. + * @phys: The physical address to map. + * @size: The size of the range to map. + * + * The hypervisor VA @haddr must have been first allocated with + * pkvm_alloc_private_va_range(). The created mapping can later be removed + * with pkvm_remove_mappings(). + * + * Return: 0 on success, negative error code on failure. + */ +int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size) +{ + unsigned long addr = (unsigned long)haddr; + int ret; + + if (!PAGE_ALIGNED(addr | phys | size)) + return -EINVAL; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + + if (addr < __io_map_base || addr >= __io_map_next) + return -EINVAL; + + if (size > __io_map_next - addr) + return -EINVAL; + + ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, PAGE_HYP); + if (ret) + kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size); + + return ret; +} + +/** + * pkvm_private_va_range_pa - Translate a private VA to a physical address. + * @va: The private virtual address to translate. + * + * Return: The physical address corresponding to @va. + */ +phys_addr_t pkvm_private_va_range_pa(void *va) +{ + kvm_pte_t pte; + s8 level; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + WARN_ON(kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)va, &pte, &level, 0)); + WARN_ON(!kvm_pte_valid(pte)); + + return kvm_pte_to_phys(pte) + ((u64)va & (kvm_granule_size(level) - 1)); +} + int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr) @@ -147,6 +200,30 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) return ret; } +/** + * pkvm_remove_mappings - Remove mappings from the hypervisor page-table + * @from: The starting virtual address of the range to remove + * @to: The ending virtual address of the range to remove + * + * The range [@from, @to) can be a subset of the range previously mapped by + * pkvm_create_mappings(), as the latter enforces PTE-level mappings. However, + * it cannot be a subset of a range mapped by pkvm_map_private_va_range() as + * that function allows block mappings which the hypervisor page-table does not + * split. + */ +void pkvm_remove_mappings(void *from, void *to) +{ + u64 size; + + WARN_ON(from > to); + to = PTR_ALIGN(to, PAGE_SIZE); + from = PTR_ALIGN_DOWN(from, PAGE_SIZE); + size = (u64)to - (u64)from; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, (u64)from, size) != size); +} + int hyp_back_vmemmap(phys_addr_t back) { unsigned long i, start, size, end = 0; @@ -486,7 +563,7 @@ static int check_page_ownership(phys_addr_t phys) return -EPERM; } - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL, 0); if (ret) return ret;
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 6290c4b..5168321 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -5,17 +5,25 @@ */ #include <kvm/arm_hypercalls.h> +#include <kvm/arm_psci.h> #include <linux/kvm_host.h> #include <linux/mm.h> #include <asm/kvm_emulate.h> +#include <asm/spectre.h> +#include <nvhe/alloc.h> #include <nvhe/mem_protect.h> #include <nvhe/memory.h> #include <nvhe/pkvm.h> #include <nvhe/trap_handler.h> +/* The host's Spectre mitigation state, for SMCCC_ARCH_FEATURES. */ +enum mitigation_state spectre_v2_state; +enum mitigation_state spectre_v4_state; +enum mitigation_state spectre_bhb_state; + /* Used by icache_is_aliasing(). */ unsigned long __icache_flags; @@ -109,7 +117,7 @@ static void pvm_init_traps_mdcr(struct kvm_vcpu *vcpu) if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) { val |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; - val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME | MDCR_EL2_HPMN_MASK); + val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME); } if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP)) @@ -285,7 +293,7 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, } hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu); - hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_inc(hyp_vm); unlock: hyp_spin_unlock(&vm_table_lock); @@ -301,7 +309,7 @@ void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_spin_lock(&vm_table_lock); hyp_vcpu->loaded_hyp_vcpu = NULL; __this_cpu_write(loaded_hyp_vcpu, NULL); - hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_dec(hyp_vm); hyp_spin_unlock(&vm_table_lock); } @@ -311,6 +319,43 @@ struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void) } +static struct pkvm_hyp_vm *pkvm_get_loaded_hyp_vm(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + + if (hyp_vcpu && + (vcpu == &hyp_vcpu->vcpu || vcpu == hyp_vcpu->host_vcpu)) + return pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + + return NULL; +} + +/* + * A loaded vCPU's VM is the hyp VM. An unloaded host vCPU's is mapped at + * EL2 only while pinned, so it's read once and pinned; the pin fails for + * memory the host isn't sharing. + */ +struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_get_loaded_hyp_vm(vcpu); + struct kvm *kvm; + + if (hyp_vm) + return &hyp_vm->kvm; + + kvm = kern_hyp_va(READ_ONCE(vcpu->kvm)); + if (hyp_pin_shared_mem(kvm, kvm + 1)) + return NULL; + + return kvm; +} + +void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ + if (kvm && !pkvm_get_loaded_hyp_vm(vcpu)) + hyp_unpin_shared_mem(kvm, kvm + 1); +} + struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) { struct pkvm_hyp_vm *hyp_vm; @@ -318,7 +363,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) hyp_spin_lock(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); if (hyp_vm) - hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_inc(hyp_vm); hyp_spin_unlock(&vm_table_lock); return hyp_vm; @@ -327,7 +372,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) void put_pkvm_hyp_vm(struct pkvm_hyp_vm *hyp_vm) { hyp_spin_lock(&vm_table_lock); - hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_dec(hyp_vm); hyp_spin_unlock(&vm_table_lock); } @@ -373,21 +418,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED); - bitmap_zero(allowed_features, KVM_VCPU_MAX_FEATURES); - - set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) - set_bit(KVM_ARM_VCPU_PMU_V3, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) - set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) - set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) - set_bit(KVM_ARM_VCPU_SVE, allowed_features); + kvm_pkvm_vcpu_allowed_features(kvm, allowed_features); bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, allowed_features, KVM_VCPU_MAX_FEATURES); @@ -396,6 +427,40 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); } +static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu, u32 mp_state) +{ + struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + struct kvm_vcpu *host_vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + /* The host manages a non-protected vCPU: always ON at EL2. */ + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON; + return 0; + } + + if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) + return -EINVAL; + + if (mp_state == KVM_MP_STATE_STOPPED) { + reset_state->reset = false; + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF; + return 0; + } + + hyp_assert_lock_held(&vm_table_lock); + if (hyp_vm->primary_vcpu) + return -EINVAL; + hyp_vm->primary_vcpu = hyp_vcpu; + + host_vcpu = hyp_vcpu->host_vcpu; + reset_state->pc = READ_ONCE(*vcpu_pc(host_vcpu)); + reset_state->r0 = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + reset_state->reset = true; + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; + return 0; +} + static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) { if (host_vcpu) @@ -440,7 +505,11 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; - hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); + if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM) + hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM; + else + hyp_vm->kvm.arch.vm_flavor = VM_PKVM; + hyp_vm->kvm.arch.flags = 0; pkvm_init_features_from_host(hyp_vm, host_kvm); @@ -518,10 +587,12 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *host_vcpu) { int ret = 0; + u32 mp_state; if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) return -EBUSY; + mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); hyp_vcpu->host_vcpu = host_vcpu; hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; @@ -530,7 +601,6 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); - hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { /* @@ -555,9 +625,17 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, goto done; ret = pkvm_vcpu_init_sve(hyp_vcpu, host_vcpu); -done: if (ret) + goto done; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); + ret = pkvm_vcpu_init_psci(hyp_vcpu, mp_state); +done: + if (ret) { unpin_host_vcpu(host_vcpu); + unpin_host_sve_state(hyp_vcpu); + } return ret; } @@ -661,9 +739,9 @@ static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); } -static void *map_donated_memory_noclear(unsigned long host_va, size_t size) +static void *map_donated_memory_noclear(void __kern *host_va, size_t size) { - void *va = (void *)kern_hyp_va(host_va); + void *va = kern_hyp_va_host(host_va); if (!PAGE_ALIGNED(va)) return NULL; @@ -675,16 +753,6 @@ static void *map_donated_memory_noclear(unsigned long host_va, size_t size) return va; } -static void *map_donated_memory(unsigned long host_va, size_t size) -{ - void *va = map_donated_memory_noclear(host_va, size); - - if (va) - memset(va, 0, size); - - return va; -} - static void __unmap_donated_memory(void *va, size_t size) { kvm_flush_dcache_to_poc(va, size); @@ -787,7 +855,7 @@ struct pkvm_hyp_vcpu *init_selftest_vm(void *virt) continue; p[i].refcount = 1; if (seeded < min_pages) { - push_hyp_memcache(&selftest_vcpu.vcpu.arch.pkvm_memcache, + push_hyp_memcache(&selftest_vcpu.vcpu.arch.stage2_mc, hyp_page_to_virt(&p[i]), hyp_virt_to_phys); seeded++; } else { @@ -814,16 +882,13 @@ void teardown_selftest_vm(void) * Unmap the donated memory from the host at stage 2. * * host_kvm: A pointer to the host's struct kvm. - * vm_hva: The host va of the area being donated for the VM state. - * Must be page aligned. * pgd_hva: The host va of the area being donated for the stage-2 PGD for * the VM. Must be page aligned. Its size is implied by the VM's * VTCR. * * Return 0 success, negative error code on failure. */ -int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva) +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva) { struct pkvm_hyp_vm *hyp_vm = NULL; size_t vm_size, pgd_size; @@ -851,15 +916,17 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, vm_size = pkvm_get_hyp_vm_size(nr_vcpus); pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr); - ret = -ENOMEM; + hyp_vm = hyp_alloc(vm_size); + if (!hyp_vm) { + ret = hyp_alloc_errno(); + goto err_unpin_kvm; + } - hyp_vm = map_donated_memory(vm_hva, vm_size); - if (!hyp_vm) - goto err_remove_mappings; + ret = -ENOMEM; pgd = map_donated_memory_noclear(pgd_hva, pgd_size); if (!pgd) - goto err_remove_mappings; + goto err_free_hyp_vm; init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle); @@ -877,8 +944,9 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, err_destroy_stage2: kvm_guest_destroy_stage2(hyp_vm); err_remove_mappings: - unmap_donated_memory(hyp_vm, vm_size); unmap_donated_memory(pgd, pgd_size); +err_free_hyp_vm: + hyp_free(hyp_vm); err_unpin_kvm: hyp_unpin_shared_mem(host_kvm, host_kvm + 1); return ret; @@ -913,16 +981,15 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, return 0; } -int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - unsigned long vcpu_hva) +int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct pkvm_hyp_vm *hyp_vm; int ret; - hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu)); + hyp_vcpu = hyp_alloc(sizeof(*hyp_vcpu)); if (!hyp_vcpu) - return -ENOMEM; + return hyp_alloc_errno(); hyp_spin_lock(&vm_table_lock); @@ -934,33 +1001,30 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); if (ret) - goto unlock; + goto unclaim; ret = register_hyp_vcpu(hyp_vm, hyp_vcpu); if (ret) { unpin_host_vcpu(host_vcpu); unpin_host_sve_state(hyp_vcpu); + goto unclaim; } + goto unlock; +unclaim: + /* + * Under vm_table_lock, so no other claim can have landed: undo + * this one. + */ + if (hyp_vm->primary_vcpu == hyp_vcpu) + hyp_vm->primary_vcpu = NULL; unlock: hyp_spin_unlock(&vm_table_lock); - if (ret) - unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu)); + hyp_free(hyp_vcpu); + return ret; } -static void -teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size) -{ - size = PAGE_ALIGN(size); - memset(addr, 0, size); - - for (void *start = addr; start < addr + size; start += PAGE_SIZE) - push_hyp_memcache(mc, start, hyp_virt_to_phys); - - unmap_donated_memory_noclear(addr, size); -} - int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn) { struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); @@ -983,7 +1047,7 @@ static struct pkvm_hyp_vm *get_pkvm_unref_hyp_vm_locked(pkvm_handle_t handle) hyp_assert_lock_held(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); - if (!hyp_vm || hyp_page_count(hyp_vm)) + if (!hyp_vm || hyp_vm->refcount) return NULL; return hyp_vm; @@ -1010,11 +1074,10 @@ int __pkvm_start_teardown_vm(pkvm_handle_t handle) int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) { - struct kvm_hyp_memcache *mc, *stage2_mc; + struct kvm_hyp_memcache *stage2_mc; struct pkvm_hyp_vm *hyp_vm; struct kvm *host_kvm; unsigned int idx; - size_t vm_size; int err; hyp_spin_lock(&vm_table_lock); @@ -1032,12 +1095,11 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) hyp_spin_unlock(&vm_table_lock); /* Reclaim guest pages (including page-table pages) */ - mc = &host_kvm->arch.pkvm.teardown_mc; stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc; reclaim_pgtable_pages(hyp_vm, stage2_mc); unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); - /* Push the metadata pages to the teardown memcache */ + /* Push the stage-2 pages to the teardown memcache */ for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; struct kvm_hyp_memcache *vcpu_mc; @@ -1045,7 +1107,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) if (!hyp_vcpu) continue; - vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache; + vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; while (vcpu_mc->nr_pages) { void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt); @@ -1054,11 +1116,10 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) unmap_donated_memory_noclear(addr, PAGE_SIZE); } - teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); + hyp_free(hyp_vcpu); } - vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus); - teardown_donated_memory(mc, hyp_vm, vm_size); + hyp_free(hyp_vm); hyp_unpin_shared_mem(host_kvm, host_kvm + 1); return 0; @@ -1133,6 +1194,238 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) } /* + * Reset the vCPU to its power-on state and commit ON_PENDING -> ON, on the + * target's own CPU. Returns -ECANCELED, with no side effects, if a rollback + * raced the reset. + * + * The requestor's PSCI CPU_ON, in pvm_psci_vcpu_on(): + * + * cmpxchg_relaxed(power_state): OFF -> ON_PENDING + * <ctrl> + * write_reset_state(); + * smp_store_release(reset_state->reset, true); + * + * The target coming online here, running, then its PSCI CPU_OFF in + * pvm_psci_vcpu_off(): + * + * smp_load_acquire(reset_state->reset) == true; + * cmpxchg_relaxed(power_state): ON_PENDING -> ON; + * read_reset_state(); + * + * <run guest> + * + * WRITE_ONCE(reset_state->reset, false); + * smp_store_release(power_state, OFF); + */ +int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; + + /* + * Pairs with smp_store_release(&reset_state->reset, true) in + * pvm_psci_vcpu_on(). The acquire must precede the cmpxchg: reversed, a + * winning cmpxchg with a false acquire would leave power_state == ON + * with the reset skipped. + */ + if (!smp_load_acquire(&reset_state->reset)) + return -ECANCELED; + + if (cmpxchg_relaxed(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, + PSCI_0_2_AFFINITY_LEVEL_ON) != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING) + return -ECANCELED; + + kvm_reset_vcpu_core(&hyp_vcpu->vcpu); + kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); + + /* Must be done after resetting sys registers. */ + kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state); + + hyp_vcpu->exit_code = 0; + return 0; +} + +struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, unsigned long mpidr) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + int i; + + mpidr &= MPIDR_HWID_BITMASK; + + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { + /* Pairs with smp_store_release() in register_hyp_vcpu(). */ + hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[i]); + + if (!hyp_vcpu) + continue; + + if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) + return hyp_vcpu; + } + + return NULL; +} + +/* + * Returns true when handled at EL2, false when the host must wake the target + * vCPU. + */ +static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + struct vcpu_reset_state *reset_state; + struct pkvm_hyp_vcpu *target; + unsigned long cpu_id, ret; + int power_state; + + cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu); + if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) { + ret = PSCI_RET_INVALID_PARAMS; + goto error; + } + + target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); + if (!target) { + ret = PSCI_RET_INVALID_PARAMS; + goto error; + } + + /* + * vCPUs race to power on the same target. The writes below depend on + * the cmpxchg reading OFF, and that control dependency with the release + * of OFF in pvm_psci_vcpu_off() orders them after the target's accesses + * to reset_state, so relaxed suffices. + */ + power_state = cmpxchg_relaxed(&target->power_state, + PSCI_0_2_AFFINITY_LEVEL_OFF, + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING); + switch (power_state) { + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + ret = PSCI_RET_ON_PENDING; + goto error; + case PSCI_0_2_AFFINITY_LEVEL_ON: + ret = PSCI_RET_ALREADY_ON; + goto error; + case PSCI_0_2_AFFINITY_LEVEL_OFF: + break; + default: + ret = PSCI_RET_INTERNAL_FAILURE; + goto error; + } + + reset_state = &target->vcpu.arch.reset_state; + WRITE_ONCE(reset_state->pc, smccc_get_arg2(&hyp_vcpu->vcpu)); + WRITE_ONCE(reset_state->r0, smccc_get_arg3(&hyp_vcpu->vcpu)); + WRITE_ONCE(reset_state->be, kvm_vcpu_is_be(&hyp_vcpu->vcpu)); + /* + * Publish reset_state.{pc, r0, be} to the target vCPU. Pairs with + * smp_load_acquire(&reset_state->reset) in pkvm_reset_vcpu(). + */ + smp_store_release(&reset_state->reset, true); + + /* The host requests KVM_REQ_VCPU_RESET and wakes the target. */ + return false; + +error: + smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0); + return true; +} + +/* + * Returns true when handled at EL2, false when the host must stop scheduling + * the vCPU. + */ +static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + /* No other writer runs while this vCPU is ON and executing. */ + WARN_ON(READ_ONCE(hyp_vcpu->power_state) != PSCI_0_2_AFFINITY_LEVEL_ON); + + WRITE_ONCE(hyp_vcpu->vcpu.arch.reset_state.reset, false); + + /* + * Orders pkvm_reset_vcpu()'s accesses to reset_state, and the clear + * above, before OFF. Pairs with the control dependency in + * pvm_psci_vcpu_on(). + */ + smp_store_release(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF); + + /* Return to the host so that it can finish powering off the vcpu. */ + return false; +} + +static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + /* Nothing to be handled by the host. Go back to the guest. */ + smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0); + return true; +} + +static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u32 feature = smccc_get_arg1(vcpu); + unsigned long val; + + switch (feature) { + case PSCI_0_2_FN_PSCI_VERSION: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_1_0_FN_PSCI_FEATURES: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + case ARM_SMCCC_VERSION_FUNC_ID: + val = PSCI_RET_SUCCESS; + break; + default: + val = PSCI_RET_NOT_SUPPORTED; + break; + } + + /* Nothing to be handled by the host. Go back to the guest. */ + smccc_set_retval(vcpu, val, 0, 0, 0); + return true; +} + +static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u32 psci_fn = smccc_get_function(vcpu); + + switch (psci_fn) { + case PSCI_0_2_FN_CPU_ON: + kvm_psci_narrow_to_32bit(vcpu); + fallthrough; + case PSCI_0_2_FN64_CPU_ON: + return pvm_psci_vcpu_on(hyp_vcpu); + case PSCI_0_2_FN_CPU_OFF: + return pvm_psci_vcpu_off(hyp_vcpu); + case PSCI_0_2_FN_PSCI_VERSION: + return pvm_psci_version(hyp_vcpu); + case PSCI_1_0_FN_PSCI_FEATURES: + return pvm_psci_features(hyp_vcpu); + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + return false; /* Handled by the host. */ + default: + /* Unknown PSCI calls are handled here, not forwarded. */ + smccc_set_retval(vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0); + return true; + } +} + +/* * Handler for protected VM HVC calls. * * Returns true if the hypervisor has handled the exit (and control @@ -1141,10 +1434,73 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) */ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) { + struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); u64 val[4] = { SMCCC_RET_INVALID_PARAMETER }; bool handled = true; + u32 feature; + uuid_t uuid; switch (smccc_get_function(vcpu)) { + case ARM_SMCCC_VERSION_FUNC_ID: + /* Nothing to be handled by the host. Go back to the guest. */ + val[0] = ARM_SMCCC_VERSION_1_1; + break; + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + /* The workaround calls themselves are handled on hyp entry. */ + val[0] = SMCCC_RET_NOT_SUPPORTED; + feature = smccc_get_arg1(vcpu); + switch (feature) { + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + val[0] = SMCCC_RET_SUCCESS; + break; + case ARM_SMCCC_ARCH_WORKAROUND_1: + switch (spectre_v2_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; + case ARM_SMCCC_ARCH_WORKAROUND_2: + switch (spectre_v4_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + /* With SSBS advertised the guest's default is safe. */ + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP)) + break; + fallthrough; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_RET_NOT_REQUIRED; + break; + } + break; + case ARM_SMCCC_ARCH_WORKAROUND_3: + switch (spectre_bhb_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; + } + break; + case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: + uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM; + val[0] = smccc_uuid_to_reg(&uuid, 0); + val[1] = smccc_uuid_to_reg(&uuid, 1); + val[2] = smccc_uuid_to_reg(&uuid, 2); + val[3] = smccc_uuid_to_reg(&uuid, 3); + break; case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES); val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); @@ -1177,8 +1533,7 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) pkvm_memunshare_call(val, vcpu); break; default: - /* Punt everything else back to the host, for now. */ - handled = false; + return pkvm_handle_psci(hyp_vcpu); } if (handled)
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 45ac5f2..7bbc26b 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -10,6 +10,7 @@ #include <asm/kvm_pgtable.h> #include <asm/kvm_pkvm.h> +#include <nvhe/alloc.h> #include <nvhe/early_alloc.h> #include <nvhe/ffa.h> #include <nvhe/gfp.h> @@ -377,6 +378,11 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_bas if (ret) return ret; + /* Arbitrary size, unmapped at boot and only consume VA-space */ + ret = hyp_alloc_init(SZ_128M); + if (ret) + return ret; + update_nvhe_init_params(); /* Jump in the idmap page to switch to the new page-tables */
diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c index 7c832d6..11fadbe 100644 --- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c +++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
@@ -8,6 +8,7 @@ #include <asm/kvm_hyp.h> #include <asm/memory.h> #include <asm/percpu.h> +#include <asm/stacktrace/nvhe.h> DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) __aligned(16); @@ -35,8 +36,6 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc) } #ifdef CONFIG_PKVM_STACKTRACE -#include <asm/stacktrace/nvhe.h> - DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace); static struct stack_info stackinfo_get_overflow(void)
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 7318e3e..91e1edf 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -203,15 +203,36 @@ static const exit_handler_fn hyp_exit_handlers[] = { [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops, }; +/* WFI/WFE exit to the host, which emulates them. WFxT is not advertised. */ +static bool kvm_handle_pvm_wfx(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (kvm_vcpu_get_esr(vcpu) & ESR_ELx_WFx_ISS_WFxT) + return kvm_handle_pvm_restricted(vcpu, exit_code); + + return false; +} + +/* Lazy FP/SIMD switch, or the UNDEF that would otherwise be left to the host. */ +static bool kvm_handle_pvm_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (kvm_hyp_handle_fpsimd(vcpu, exit_code)) + return true; + + return kvm_handle_pvm_restricted(vcpu, exit_code); +} + +/* + * A class not listed takes an UNDEF at EL2: the host has no way to + * inject one into a protected vCPU. + */ static const exit_handler_fn pvm_exit_handlers[] = { - [0 ... ESR_ELx_EC_MAX] = NULL, + [0 ... ESR_ELx_EC_MAX] = kvm_handle_pvm_restricted, + [ESR_ELx_EC_WFx] = kvm_handle_pvm_wfx, [ESR_ELx_EC_HVC64] = kvm_handle_pvm_hvc64, [ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64, - [ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted, - [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_fpsimd, [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, - [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low, [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops, };
diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8758c68..32e8265 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
@@ -121,6 +121,9 @@ static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = { MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN4, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN16, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN64, IMP), FEAT_END }; @@ -525,6 +528,66 @@ static const struct sys_reg_desc pvm_sys_reg_descs[] = { /* Performance Monitoring Registers are restricted. */ }; +struct sys_reg_desc_reset { + int reg; + void (*reset)(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *rd); + u64 value; +}; + +static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r) +{ + __vcpu_assign_sys_reg(vcpu, r->reg, kvm_calculate_mpidr(vcpu)); +} + +static void reset_value(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r) +{ + __vcpu_assign_sys_reg(vcpu, r->reg, r->value); +} + +#define RESET_VAL(REG, VAL) { REG, reset_value, VAL } + +#define RESET_ZERO(REG) RESET_VAL(REG, 0) + +#define RESET_UNKNOWN(REG) RESET_VAL(REG, 0x1de7ec7edbadc0deULL) + +#define RESET_FUNC(REG, FN) { REG, FN, 0 } + +static const struct sys_reg_desc_reset pvm_sys_reg_reset_vals[] = { + RESET_FUNC(MPIDR_EL1, reset_mpidr), + RESET_UNKNOWN(TPIDR_EL0), + RESET_UNKNOWN(TPIDRRO_EL0), + RESET_UNKNOWN(TPIDR_EL1), + RESET_ZERO(CNTKCTL_EL1), + RESET_UNKNOWN(PAR_EL1), + RESET_ZERO(DISR_EL1), + RESET_ZERO(CPACR_EL1), + RESET_VAL(CONTEXTIDR_EL1, 0x00000000dbadc0deULL), + RESET_VAL(SCTLR_EL1, 0x00C50078ULL), + RESET_ZERO(TCR_EL1), + RESET_UNKNOWN(AFSR0_EL1), + RESET_UNKNOWN(AFSR1_EL1), + RESET_UNKNOWN(ESR_EL1), + RESET_UNKNOWN(MAIR_EL1), + RESET_ZERO(AMAIR_EL1), + RESET_ZERO(MDSCR_EL1), + RESET_ZERO(CNTV_CTL_EL0), + RESET_UNKNOWN(TTBR0_EL1), + RESET_UNKNOWN(TTBR1_EL1), + RESET_UNKNOWN(FAR_EL1), + RESET_VAL(VBAR_EL1, 0x1de7ec7edbadc000ULL), +}; + +void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu) +{ + unsigned long i; + + for (i = 0; i < ARRAY_SIZE(pvm_sys_reg_reset_vals); i++) { + const struct sys_reg_desc_reset *r = &pvm_sys_reg_reset_vals[i]; + + r->reset(vcpu, r); + } +} + /* * Initializes feature registers for protected vms. */
diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 65be6c9..0d7e4db 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c
@@ -4,6 +4,7 @@ * Author: Vincent Donnefort <vdonnefort@google.com> */ +#include <nvhe/alloc.h> #include <nvhe/clock.h> #include <nvhe/mem_protect.h> #include <nvhe/mm.h> @@ -62,18 +63,34 @@ static void __release_host_mem(void *start, u64 size) WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT)); } -static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_buffer, - struct hyp_trace_desc *desc) +static int hyp_trace_buffer_alloc_bpages(struct hyp_trace_buffer *trace_buffer, + struct hyp_trace_desc *desc) { - void *start = (void *)kern_hyp_va(desc->bpages_backing_start); - size_t size = desc->bpages_backing_size; + void *start; + size_t size; int ret; - ret = __admit_host_mem(start, size); - if (ret) - return ret; + if (is_protected_kvm_enabled()) { + struct ring_buffer_desc *rb_desc; + int cpu; - memset(start, 0, size); + size = 0; + for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) + size += rb_desc->nr_page_va * sizeof(struct simple_buffer_page); + + start = hyp_alloc(size); + if (!start) + return hyp_alloc_errno(); + } else { + start = (void *)kern_hyp_va(desc->bpages_backing_start); + size = desc->bpages_backing_size; + + ret = __admit_host_mem(start, size); + if (ret) + return ret; + + memset(start, 0, size); + } trace_buffer->bpages_backing_start = start; trace_buffer->bpages_backing_size = size; @@ -81,7 +98,7 @@ static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_bu return 0; } -static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace_buffer) +static void hyp_trace_buffer_free_bpages(struct hyp_trace_buffer *trace_buffer) { void *start = trace_buffer->bpages_backing_start; size_t size = trace_buffer->bpages_backing_size; @@ -89,11 +106,14 @@ static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace if (!size) return; - memset(start, 0, size); + if (is_protected_kvm_enabled()) { + hyp_free(start); + } else { + memset(start, 0, size); + __release_host_mem(start, size); + } - __release_host_mem(start, size); - - trace_buffer->bpages_backing_start = 0; + trace_buffer->bpages_backing_start = NULL; trace_buffer->bpages_backing_size = 0; } @@ -128,7 +148,7 @@ static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer) simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu), __unpin_shared_page); - hyp_trace_buffer_unload_bpage_backing(trace_buffer); + hyp_trace_buffer_free_bpages(trace_buffer); } static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, @@ -143,7 +163,7 @@ static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, if (hyp_trace_buffer_loaded(trace_buffer)) return -EINVAL; - ret = hyp_trace_buffer_load_bpage_backing(trace_buffer, desc); + ret = hyp_trace_buffer_alloc_bpages(trace_buffer, desc); if (ret) return ret; @@ -168,47 +188,43 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz { struct ring_buffer_desc *rb_desc; unsigned int cpu; - size_t nr_bpages; void *desc_end; if (!is_protected_kvm_enabled()) return true; /* - * Both desc_size and bpages_backing_size are untrusted host-provided - * values. We rely on __pkvm_host_donate_hyp() to enforce their validity. + * desc_size is an untrusted host-provided value. We rely on + * __pkvm_host_donate_hyp() to enforce its validity. */ desc_end = (void *)desc + desc_size; - nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page); + + if (desc->trace_buffer_desc.nr_cpus != hyp_nr_cpus) + return false; for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) { /* Can we read nr_page_va? */ if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end) return false; + /* simple_ring_buffer_init_mm() expects at least 3 pages */ + if (rb_desc->nr_page_va < 3) + return false; + /* Overflow desc? */ if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end) return false; - /* Overflow bpages backing memory? */ - if (nr_bpages < rb_desc->nr_page_va) - return false; - - if (cpu >= hyp_nr_cpus) - return false; - if (cpu != rb_desc->cpu) return false; - - nr_bpages -= rb_desc->nr_page_va; } return true; } -int __tracing_load(unsigned long desc_hva, size_t desc_size) +int __tracing_load(void __kern *desc_hva, size_t desc_size) { - struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva); + struct hyp_trace_desc *desc = kern_hyp_va_host(desc_hva); int ret; ret = __admit_host_mem(desc, desc_size);
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b74dd5c..9c83837 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -8,6 +8,7 @@ */ #include <linux/bitfield.h> +#include <linux/kvm_host.h> #include <asm/kvm_pgtable.h> #include <asm/stage2_pgtable.h> @@ -298,12 +299,13 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx, } int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, s8 *level) + kvm_pte_t *ptep, s8 *level, + enum kvm_pgtable_walk_flags flags) { struct leaf_walk_data data; struct kvm_pgtable_walker walker = { .cb = leaf_walker, - .flags = KVM_PGTABLE_WALK_LEAF, + .flags = flags | KVM_PGTABLE_WALK_LEAF, .arg = &data, }; int ret;
diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c index 6d69dfe..9474631 100644 --- a/arch/arm64/kvm/hyp/vgic-v5-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c
@@ -7,6 +7,41 @@ #include <asm/kvm_hyp.h> +void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if) +{ + write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2); + isb(); + + /* Catch any faults */ + cpu_if->vgic_contextr = read_sysreg_s(SYS_ICH_CONTEXTR_EL2); + if (!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr)) + return; + + cpu_if->gicv5_vpe.resident = true; +} + +void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if) +{ + /* + * Clear the db_fired state to ensure that we're ready for the next + * doorbell when it is requested. If a doorbell firing caused us to + * enter the guest, then we've already consumed that state at this + * point, so this is safe to clear. Use WRITE_ONCE() to ensure we're not + * racing with the doorbell firing and setting the state true again. + */ + WRITE_ONCE(cpu_if->gicv5_vpe.db_fired, false); + + /* + * Make as non-resident before actually making non-resident. Avoids race + * with doorbell arriving. + */ + cpu_if->gicv5_vpe.resident = false; + dsb(st); + + write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2); + isb(); +} + void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if) { cpu_if->vgic_apr = read_sysreg_s(SYS_ICH_APR_EL2); @@ -114,3 +149,13 @@ void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if) { write_sysreg_s(cpu_if->vgic_icsr, SYS_ICC_ICSR_EL1); } + +void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm) +{ + u64 value; + + value = intid & (GICV5_GIC_VDPEND_ID_MASK | GICV5_GIC_VDPEND_TYPE_MASK); + value |= FIELD_PREP(GICV5_GIC_VDPEND_PENDING_MASK, pending); + value |= FIELD_PREP(GICV5_GIC_VDPEND_VM_MASK, vm); + gic_insn(value, VDPEND); +}
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 7875911..4c965f9 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -175,7 +175,8 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) offset = read_sysreg_s(SYS_CNTPOFF_EL2); if (map.direct_ptimer && offset) { - write_sysreg_el0(val + offset, SYS_CNTP_CVAL); + val = timer_apply_offset(val, offset, arch_timer_read_cntpct_el0()); + write_sysreg_el0(val, SYS_CNTP_CVAL); isb(); } } @@ -296,10 +297,10 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) break; case SYS_CNTP_CVAL_EL0: if (vcpu_el2_e2h_is_set(vcpu)) { - val = read_sysreg_el0(SYS_CNTP_CVAL); - - if (!has_cntpoff()) - val -= timer_get_offset(vcpu_hptimer(vcpu)); + if (!has_cntpoff() && timer_get_offset(vcpu_hptimer(vcpu))) + val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); + else + val = read_sysreg_el0(SYS_CNTP_CVAL); } else { val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); } @@ -321,10 +322,14 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); break; case SYS_CNTV_CVAL_EL0: - if (vcpu_el2_e2h_is_set(vcpu)) - val = read_sysreg_el0(SYS_CNTV_CVAL); - else + if (vcpu_el2_e2h_is_set(vcpu)) { + if (has_broken_cntvoff() && timer_get_offset(vcpu_hvtimer(vcpu))) + val = __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2); + else + val = read_sysreg_el0(SYS_CNTV_CVAL); + } else { val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); + } break; case SYS_CNTVCT_EL0: case SYS_CNTVCTSS_EL0:
diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 9644c42..b940449 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c
@@ -13,6 +13,7 @@ #include <asm/kvm_host.h> #include <asm/kvm_hyptrace.h> #include <asm/kvm_mmu.h> +#include <asm/kvm_pkvm.h> #include "hyp_trace.h" @@ -59,14 +60,8 @@ static void __hyp_clock_work(struct work_struct *work) if (hyp_clock->mult) { u64 err, cur = delta_cycles; - if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) { - __uint128_t tmp = (__uint128_t)cur * hyp_clock->mult; - - cur = tmp >> hyp_clock->shift; - } else { - cur *= hyp_clock->mult; - cur >>= hyp_clock->shift; - } + WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64); + cur = mul_u64_u32_shr(cur, hyp_clock->mult, hyp_clock->shift); cur += hyp_clock->boot; err = abs_diff(cur, boot); @@ -84,7 +79,7 @@ static void __hyp_clock_work(struct work_struct *work) err / NSEC_PER_USEC); } - rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot); + rate = mul_u64_u64_div_u64(delta_cycles, NSEC_PER_SEC, delta_boot); clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift, rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S); @@ -151,11 +146,18 @@ static void __unshare_page(unsigned long va) static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size) { - int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; + unsigned long nr_bpages; size_t backing_size; void *start; int ret; + if (is_protected_kvm_enabled()) { + trace_buffer->desc->bpages_backing_start = 0; + trace_buffer->desc->bpages_backing_size = 0; + return 0; + } + + nr_bpages = __calc_nr_pages_ring_buffer_desc(size); backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * num_possible_cpus()); @@ -177,6 +179,9 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) { + if (!trace_buffer->desc->bpages_backing_start) + return; + free_pages_exact((void *)trace_buffer->desc->bpages_backing_start, trace_buffer->desc->bpages_backing_size); } @@ -269,7 +274,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) if (ret) goto err_free_buffer; - ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size); + ret = pkvm_call_hyp_req(__tracing_load, desc, desc_size); if (ret) goto err_unload_pages; @@ -402,7 +407,10 @@ static struct trace_remote_callbacks trace_remote_callbacks = { static const char *__hyp_enter_exit_reason_str(u8 reason); -#include <asm/kvm_define_hypevents.h> +struct remote_event_format_hyp_printk; +static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq); + +#include "define_hypevents.h" static const char *__hyp_enter_exit_reason_str(u8 reason) { @@ -421,6 +429,61 @@ static const char *__hyp_enter_exit_reason_str(u8 reason) return strs[min(reason, HYP_REASON_UNKNOWN)]; } +static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq) +{ + const struct hyp_string_fmt *hyp_str_fmt; + const char *fmt; + + trace_seq_putc(seq, ' '); + + if (entry->fmt_id >= (__hyp_string_fmts_end - __hyp_string_fmts_start)) { + trace_seq_printf(seq, "Unknown hyp_string_fmt ID %u\n", entry->fmt_id); + return; + } + + hyp_str_fmt = &__hyp_string_fmts_start[entry->fmt_id]; + fmt = hyp_str_fmt->fmt; + + switch (hyp_str_fmt->nr_args) { + case 0: + trace_seq_puts(seq, fmt); + break; + case 1: + trace_seq_printf(seq, fmt, entry->args[0]); + break; + case 2: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1]); + break; + case 3: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2]); + break; + case 4: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3]); + break; + case 5: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4]); + break; + case 6: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4], entry->args[5]); + break; + case 7: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4], entry->args[5], entry->args[6]); + break; + default: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], + entry->args[2], entry->args[3], entry->args[4], entry->args[5], + entry->args[6], entry->args[7]); + break; + } + + if (trace_seq_used(seq) && seq->buffer[trace_seq_used(seq) - 1] != '\n') + trace_seq_putc(seq, '\n'); +} + static void __init hyp_trace_init_events(void) { struct hyp_event_id *hyp_event_id = __hyp_event_ids_start; @@ -430,6 +493,9 @@ static void __init hyp_trace_init_events(void) /* Events on both sides hypervisor are sorted */ for (; event < __hyp_events_end; event++, hyp_event_id++, id++) event->id = hyp_event_id->id = id; + + WARN(__hyp_string_fmts_end - __hyp_string_fmts_start > U16_MAX + 1, + "Too many trace_hyp_printk() callsites\n"); } int __init kvm_hyp_trace_init(void)
diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index dfa25bb..f4b932b 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c
@@ -294,6 +294,10 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: feature = smccc_get_arg1(vcpu); switch (feature) { + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + val[0] = SMCCC_RET_SUCCESS; + break; case ARM_SMCCC_ARCH_WORKAROUND_1: switch (arm64_get_spectre_v2_state()) { case SPECTRE_VULNERABLE: @@ -581,6 +585,13 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) wants_02 = vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2); + /* + * EL2 advertises PSCI 1.1; the host handles the forwarded calls + * by this version. + */ + if (vcpu_is_protected(vcpu) && val < KVM_ARM_PSCI_1_1) + return -EINVAL; + switch (val) { case KVM_ARM_PSCI_0_1: if (wants_02)
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index d6c4fc1..96bf926 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c
@@ -371,15 +371,15 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) } /* - * Emulate the exception entry if SErrors are unmasked. This is useful if - * the vCPU is in a nested context w/ vSErrors enabled then we've already - * delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2, - * VDISR_EL2) to the guest hypervisor. + * With NV, emulate the exception entry if SErrors are unmasked: in a + * nested context with vSErrors enabled, the hardware vSError context + * (i.e. HCR_EL2.VSE, VSESR_EL2, VDISR_EL2) is delegated to the guest + * hypervisor. Otherwise, HCR_EL2.VSE delivers it once it is unmasked. * * As we're emulating the SError injection we need to explicitly populate * ESR_ELx.EC because hardware will not do it on our behalf. */ - if (!serror_is_masked(vcpu)) { + if (vcpu_has_nv(vcpu) && !serror_is_masked(vcpu)) { pend_serror_exception(vcpu); esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index d1c3a35..9cd1d3d 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c
@@ -6,10 +6,12 @@ #include <linux/kvm_host.h> #include <asm/kvm_emulate.h> +#include <asm/kvm_mmu.h> #include <trace/events/kvm.h> #include "trace.h" +#if ARM64_S390_COMMON == 1 void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data) { void *datap = NULL; @@ -258,3 +260,4 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) run->exit_reason = KVM_EXIT_MMIO; return 0; } +#endif /* ARM64_S390_COMMON == 1 */
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index bf6d652..745cb86 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c
@@ -5,8 +5,10 @@ */ #include <linux/acpi.h> +#include <linux/cleanup.h> #include <linux/mman.h> #include <linux/kvm_host.h> +#include <linux/interval_tree.h> #include <linux/io.h> #include <linux/hugetlb.h> #include <linux/sched/signal.h> @@ -37,6 +39,8 @@ static unsigned long __ro_after_init io_map_base; #define KVM_PGT_FN(fn) (!is_protected_kvm_enabled() ? fn : p ## fn) +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm); + static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end, phys_addr_t size) { @@ -175,6 +179,18 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); } +static int pkvm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); + return 0; +} + +static int kvm_vm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); + return 0; +} + /** * kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8 * @kvm: pointer to kvm structure. @@ -183,26 +199,31 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) */ int kvm_arch_flush_remote_tlbs(struct kvm *kvm) { - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); - return 0; + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs(kvm); } -int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, - gfn_t gfn, u64 nr_pages) +static int pkvm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return pkvm_flush_remote_tlbs(kvm); +} + +static int kvm_vm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) { u64 size = nr_pages << PAGE_SHIFT; u64 addr = gfn << PAGE_SHIFT; - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); + kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); return 0; } +int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range(kvm, gfn, nr_pages); +} + static void *stage2_memcache_zalloc_page(void *arg) { struct kvm_mmu_memory_cache *mc = arg; @@ -298,6 +319,37 @@ static void invalidate_icache_guest_page(void *va, size_t size) __invalidate_icache_guest_page(va, size); } +static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, + u64 addr, u64 size); + +static void kvm_vm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + int (*fn)(struct kvm_pgtable *, u64, u64); + + if (kvm_is_nested_s2_mmu(kvm, mmu)) + fn = kvm_pgtable_stage2_unmap_tracked; + else + fn = kvm_pgtable_stage2_unmap; + + WARN_ON(stage2_apply_range(mmu, start, start + size, fn, may_block)); +} + +static void pkvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + WARN_ON(stage2_apply_range(mmu, start, start + size, + pkvm_pgtable_stage2_unmap, may_block)); +} + +static void no_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, bool may_block) +{ +} + /* * Unmapping vs dcache management: * @@ -322,37 +374,39 @@ static void invalidate_icache_guest_page(void *va, size_t size) * we then fully enforce cacheability of RAM, no matter what the guest * does. */ + +static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, u64 addr, u64 size) +{ + int ret; + + ret = kvm_pgtable_stage2_unmap(pgt, addr, size); + if (ret) + return ret; + + kvm_remove_guest_s2_mappings(pgt->mmu, addr, size); + return 0; +} + /** - * __unmap_stage2_range -- Clear stage2 page table entries to unmap a range + * kvm_stage2_unmap_range -- Clear stage2 page table entries to unmap a range * @mmu: The KVM stage-2 MMU pointer * @start: The intermediate physical base address of the range to unmap * @size: The size of the area to unmap * @may_block: Whether or not we are permitted to block * * Clear a range of stage-2 mappings, lowering the various ref-counts. Must - * be called while holding mmu_lock (unless for freeing the stage2 pgd before - * destroying the VM), otherwise another faulting VCPU may come in and mess - * with things behind our backs. + * be called while holding mmu_lock otherwise another faulting VCPU may + * come in and mess with things behind our backs. */ -static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, - bool may_block) -{ - struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - phys_addr_t end = start + size; - - lockdep_assert_held_write(&kvm->mmu_lock); - WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, end, KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); -} - void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block) { - if (kvm_vm_is_protected(kvm_s2_mmu_to_kvm(mmu))) - return; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - __unmap_stage2_range(mmu, start, size, may_block); + lockdep_assert_held_write(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + kvm->arch.vm_s2_ops->vm_stage2_unmap_range(mmu, start, size, may_block); } void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) @@ -875,7 +929,7 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) * IPI-ing threads). */ local_irq_save(flags); - ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level); + ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level, 0); local_irq_restore(flags); if (ret) @@ -992,6 +1046,12 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t int cpu, err; struct kvm_pgtable *pgt; + /* Initialize the VM ops for the VM instance for the first time */ + if (mmu == &kvm->arch.mmu) { + err = kvm_vm_init_vm_s2_ops(kvm); + if (err) + return err; + } /* * If we already have our page tables in place, and that the * MMU context is the canonical one, we have a bug somewhere, @@ -1042,6 +1102,8 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t mmu->pgd_phys = __pa(pgt->pgd); + mmu->guest_s2_mappings = RB_ROOT_CACHED; + if (kvm_is_nested_s2_mmu(kvm, mmu)) kvm_init_nested_s2_mmu(mmu); @@ -1131,10 +1193,25 @@ void stage2_unmap_vm(struct kvm *kvm) srcu_read_unlock(&kvm->srcu, idx); } +static void guest_s2_tracking_destroy(struct rb_root_cached *tree) +{ + struct kvm_guest_s2_mapping *mapping; + struct interval_tree_node *node; + + while ((node = interval_tree_iter_first(tree, 0, ULONG_MAX))) { + interval_tree_remove(node, tree); + mapping = container_of(node, struct kvm_guest_s2_mapping, + canonical); + kfree(mapping); + cond_resched(); + } +} + void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_pgtable *pgt = NULL; + struct rb_root_cached mappings_tree; write_lock(&kvm->mmu_lock); pgt = mmu->pgt; @@ -1147,12 +1224,18 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) if (kvm_is_nested_s2_mmu(kvm, mmu)) kvm_init_nested_s2_mmu(mmu); + mappings_tree = mmu->guest_s2_mappings; + mmu->guest_s2_mappings = RB_ROOT_CACHED; + write_unlock(&kvm->mmu_lock); if (pgt) { kvm_stage2_destroy(pgt); kfree(pgt); } + + if (!kvm_is_nested_s2_mmu(kvm, mmu)) + guest_s2_tracking_destroy(&mappings_tree); } static void hyp_mc_free_fn(void *addr, void *mc) @@ -1169,8 +1252,10 @@ static void *hyp_mc_alloc_fn(void *mc) { struct kvm_hyp_memcache *memcache = mc; void *addr; + gfp_t gfp; - addr = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); + gfp = memcache->flags & HYP_MEMCACHE_ACCOUNT_KMEMCG ? GFP_KERNEL_ACCOUNT : GFP_KERNEL; + addr = (void *)__get_free_page(gfp); if (addr && memcache->flags & HYP_MEMCACHE_ACCOUNT_STAGE2) kvm_account_pgtable_pages(addr, 1); @@ -1223,7 +1308,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, KVM_PGTABLE_PROT_R | (writable ? KVM_PGTABLE_PROT_W : 0); - if (is_protected_kvm_enabled()) + if (kvm_vm_hyp_is_distrusting(kvm)) return -EPERM; size += offset_in_page(guest_ipa); @@ -1362,21 +1447,53 @@ static void kvm_send_hwpoison_signal(unsigned long address, short lsb) send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); } -static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, - unsigned long hva, +struct kvm_s2_fault_desc { + struct kvm_vcpu *vcpu; + phys_addr_t fault_ipa; + struct kvm_s2_trans *nested; + struct kvm_memory_slot *memslot; + unsigned long hva; + unsigned long esr; + struct kvm_s2_mmu *mmu; +}; + +struct kvm_s2_fault_vma_info { + unsigned long mmu_seq; + long vma_pagesize; + vm_flags_t vm_flags; + unsigned long max_map_size; + struct page *page; + kvm_pfn_t pfn; + gfn_t gfn; + bool device; + bool mte_allowed; + bool is_vma_cacheable; + bool map_writable; + bool map_non_cacheable; +}; + +struct kvm_s2_fault_result { + unsigned long mapping_size; +}; + +static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s2fd, unsigned long map_size) { - gpa_t gpa_start; + struct kvm_memory_slot *memslot = s2fd->memslot; + unsigned long hva = s2fd->hva; hva_t uaddr_start, uaddr_end; + gpa_t gpa_start; size_t size; /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ if (map_size == PAGE_SIZE) return true; - /* pKVM only supports PMD_SIZE huge-mappings */ - if (is_protected_kvm_enabled() && map_size != PMD_SIZE) - return false; + /* pKVM only supports PMD_SIZE huge-mappings for non-protected VMs */ + if (is_protected_kvm_enabled()) { + if (vcpu_is_protected(s2fd->vcpu) || map_size != PMD_SIZE) + return false; + } size = memslot->npages * PAGE_SIZE; @@ -1436,9 +1553,9 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, * Returns the size of the mapping. */ static long -transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, - unsigned long hva, kvm_pfn_t *pfnp, gfn_t *gfnp) +transparent_hugepage_adjust(const struct kvm_s2_fault_desc *s2fd, kvm_pfn_t *pfnp, gfn_t *gfnp) { + struct kvm *kvm = s2fd->vcpu->kvm; kvm_pfn_t pfn = *pfnp; gfn_t gfn = *gfnp; @@ -1447,8 +1564,8 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { - int sz = get_user_mapping_size(kvm, hva); + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) { + int sz = get_user_mapping_size(kvm, s2fd->hva); if (sz < 0) return sz; @@ -1539,12 +1656,12 @@ static void *get_mmu_memcache(struct kvm_vcpu *vcpu) if (!is_protected_kvm_enabled()) return &vcpu->arch.mmu_page_cache; else - return &vcpu->arch.pkvm_memcache; + return &vcpu->arch.stage2_mc; } -static int topup_mmu_memcache(struct kvm_vcpu *vcpu, void *memcache) +static int topup_mmu_memcache(struct kvm_s2_mmu *mmu, void *memcache) { - int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu); + int min_pages = kvm_mmu_cache_min_pages(mmu); if (!is_protected_kvm_enabled()) return kvm_mmu_topup_memory_cache(memcache, min_pages); @@ -1585,55 +1702,85 @@ static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm, return prot; } -struct kvm_s2_fault_desc { - struct kvm_vcpu *vcpu; - phys_addr_t fault_ipa; - struct kvm_s2_trans *nested; - struct kvm_memory_slot *memslot; - unsigned long hva; -}; +static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_fsc_is_permission_fault(s2fd->esr); +} -static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) +static bool kvm_s2_fault_is_exec(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_abt_is_exec_fault(s2fd->esr); +} + +static bool kvm_s2_fault_is_write(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_abt_is_write_fault(s2fd->esr); +} + +static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) +{ + u64 level; + + if (!kvm_s2_fault_is_perm(s2fd)) + return 0; + level = s2fd->esr & ESR_ELx_FSC_LEVEL; + return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); +} + +static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_vma_info *s2vi); + +static gfn_t get_canonical_gfn(const struct kvm_s2_fault_desc *s2fd, + const struct kvm_s2_fault_vma_info *s2vi); + +static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_result *result) { bool write_fault, exec_fault; - bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); + bool perm_fault = kvm_s2_fault_is_perm(s2fd); enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; - struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; - unsigned long mmu_seq; - struct page *page; + struct kvm_pgtable *pgt = s2fd->mmu->pgt; + struct kvm_guest_s2_mapping *mapping = NULL; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm *kvm = s2fd->vcpu->kvm; void *memcache = NULL; - kvm_pfn_t pfn; - gfn_t gfn; + gfn_t canonical_gfn; int ret; if (!perm_fault) { memcache = get_mmu_memcache(s2fd->vcpu); - ret = topup_mmu_memcache(s2fd->vcpu, memcache); + ret = topup_mmu_memcache(s2fd->mmu, memcache); if (ret) return ret; + if (kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + mapping = kmalloc_obj(struct kvm_guest_s2_mapping, GFP_KERNEL_ACCOUNT); + if (!mapping) + return -ENOMEM; + } } - if (s2fd->nested) - gfn = kvm_s2_trans_output(s2fd->nested) >> PAGE_SHIFT; - else - gfn = s2fd->fault_ipa >> PAGE_SHIFT; + s2vi.vma_pagesize = PAGE_SIZE; + s2vi.gfn = ALIGN_DOWN(s2fd->fault_ipa, s2vi.vma_pagesize) >> PAGE_SHIFT; + canonical_gfn = get_canonical_gfn(s2fd, &s2vi); - write_fault = kvm_is_write_fault(s2fd->vcpu); - exec_fault = kvm_vcpu_trap_is_exec_fault(s2fd->vcpu); + write_fault = kvm_s2_fault_is_write(s2fd); + exec_fault = kvm_s2_fault_is_exec(s2fd); VM_WARN_ON_ONCE(write_fault && exec_fault); - mmu_seq = kvm->mmu_invalidate_seq; + s2vi.mmu_seq = kvm->mmu_invalidate_seq; /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, NULL); if (ret) { - kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, - write_fault, exec_fault, false); - return ret; + /* If result is non-NULL this is a synthetic fault. */ + if (!result) + kvm_prepare_memory_fault_exit(s2fd->vcpu, gfn_to_gpa(canonical_gfn), + s2vi.vma_pagesize, write_fault, exec_fault, + false); + kfree(mapping); } if (!(s2fd->memslot->flags & KVM_MEM_READONLY)) @@ -1649,7 +1796,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot); kvm_fault_lock(kvm); - if (mmu_invalidate_retry(kvm, mmu_seq)) { + if (mmu_invalidate_retry(kvm, s2vi.mmu_seq)) { ret = -EAGAIN; goto out_unlock; } @@ -1660,61 +1807,67 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) * PTE, which will be preserved. */ prot &= ~KVM_NV_GUEST_MAP_SZ; - ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa, + ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, gfn_to_gpa(s2vi.gfn), prot, flags); } else { - ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE, - __pfn_to_phys(pfn), prot, + ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(s2vi.gfn), + s2vi.vma_pagesize, + __pfn_to_phys(s2vi.pfn), prot, memcache, flags); + /* + * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. + * We don't know which subrange is installed, track the whole + * thing. + */ + if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + kvm_record_guest_s2_mapping(pgt->mmu, canonical_gfn << PAGE_SHIFT, + s2fd->fault_ipa, s2vi.vma_pagesize, mapping); + mapping = NULL; + } } out_unlock: - kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); + kfree(mapping); if ((prot & KVM_PGTABLE_PROT_W) && !ret) - mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); + mark_page_dirty_in_slot(kvm, s2fd->memslot, canonical_gfn); - return ret != -EAGAIN ? ret : 0; + if (ret == -EAGAIN) + return result ? ret : 0; + + if (result && !ret) + result->mapping_size = s2vi.vma_pagesize; + + return ret; } -struct kvm_s2_fault_vma_info { - unsigned long mmu_seq; - long vma_pagesize; - vm_flags_t vm_flags; - unsigned long max_map_size; - struct page *page; - kvm_pfn_t pfn; - gfn_t gfn; - bool device; - bool mte_allowed; - bool is_vma_cacheable; - bool map_writable; - bool map_non_cacheable; -}; - -static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm_vcpu *vcpu = s2fd->vcpu; - struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt; + struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct mm_struct *mm = current->mm; struct kvm *kvm = vcpu->kvm; void *hyp_memcache; - struct page *page; int ret; hyp_memcache = get_mmu_memcache(vcpu); - ret = topup_mmu_memcache(vcpu, hyp_memcache); + ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache); if (ret) return -ENOMEM; + ret = kvm_s2_fault_get_vma_info(s2fd, &s2vi); + if (ret) + return ret; + ret = account_locked_vm(mm, 1, true); if (ret) return ret; mmap_read_lock(mm); - ret = pin_user_pages(s2fd->hva, 1, flags, &page); + ret = pin_user_pages(s2fd->hva, 1, flags, &s2vi.page); mmap_read_unlock(mm); if (ret == -EHWPOISON) { @@ -1724,7 +1877,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } else if (ret != 1) { ret = -EFAULT; goto dec_account; - } else if (!folio_test_swapbacked(page_folio(page))) { + } else if (!folio_test_swapbacked(page_folio(s2vi.page))) { /* * We really can't deal with page-cache pages returned by GUP * because (a) we may trigger writeback of a page for which we @@ -1744,8 +1897,8 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } write_lock(&kvm->mmu_lock); - ret = pkvm_pgtable_stage2_map(pgt, s2fd->fault_ipa, PAGE_SIZE, - page_to_phys(page), KVM_PGTABLE_PROT_RWX, + ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), PAGE_SIZE, + page_to_phys(s2vi.page), KVM_PGTABLE_PROT_RWX, hyp_memcache, 0); write_unlock(&kvm->mmu_lock); if (ret) { @@ -1756,7 +1909,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) return 0; unpin: - unpin_user_pages(&page, 1); + unpin_user_page(s2vi.page); dec_account: account_locked_vm(mm, 1, false); return ret; @@ -1779,7 +1932,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, switch (vma_shift) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PUD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PUD_SIZE)) break; fallthrough; #endif @@ -1787,7 +1940,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, vma_shift = PMD_SHIFT; fallthrough; case PMD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PMD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) break; fallthrough; case CONT_PTE_SHIFT: @@ -1828,11 +1981,6 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, return vma_shift; } -static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) -{ - return kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); -} - static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, struct kvm_s2_fault_vma_info *s2vi) { @@ -1898,13 +2046,11 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd, return ret; s2vi->pfn = __kvm_faultin_pfn(s2fd->memslot, get_canonical_gfn(s2fd, s2vi), - kvm_is_write_fault(s2fd->vcpu) ? FOLL_WRITE : 0, + kvm_s2_fault_is_write(s2fd) ? FOLL_WRITE : 0, &s2vi->map_writable, &s2vi->page); if (unlikely(is_error_noslot_pfn(s2vi->pfn))) { - if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) { - kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi->vma_pagesize)); - return 0; - } + if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) + return -EHWPOISON; return -EFAULT; } @@ -1946,7 +2092,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, { struct kvm *kvm = s2fd->vcpu->kvm; - if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu) && s2vi->map_non_cacheable) + if (kvm_s2_fault_is_exec(s2fd) && s2vi->map_non_cacheable) return -ENOEXEC; /* @@ -1955,7 +2101,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, * and trigger the exception here. Since the memslot is valid, inject * the fault back to the guest. */ - if (esr_fsc_is_excl_atomic_fault(kvm_vcpu_get_esr(s2fd->vcpu))) { + if (esr_fsc_is_excl_atomic_fault(s2fd->esr)) { kvm_inject_dabt_excl_atomic(s2fd->vcpu, kvm_vcpu_get_hfar(s2fd->vcpu)); return 1; } @@ -1964,13 +2110,13 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, if (s2vi->map_writable && (s2vi->device || !memslot_is_logging(s2fd->memslot) || - kvm_is_write_fault(s2fd->vcpu))) + kvm_s2_fault_is_write(s2fd))) *prot |= KVM_PGTABLE_PROT_W; if (s2fd->nested) *prot = adjust_nested_fault_perms(s2fd->nested, *prot); - if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu)) + if (kvm_s2_fault_is_exec(s2fd)) *prot |= KVM_PGTABLE_PROT_X; if (s2vi->map_non_cacheable) @@ -1994,11 +2140,14 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, const struct kvm_s2_fault_vma_info *s2vi, enum kvm_pgtable_prot prot, - void *memcache) + void *memcache, + struct kvm_s2_fault_result *result) { enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; + struct kvm_guest_s2_mapping *mapping = NULL; bool writable = prot & KVM_PGTABLE_PROT_W; struct kvm *kvm = s2fd->vcpu->kvm; + phys_addr_t canonical_ipa; struct kvm_pgtable *pgt; long perm_fault_granule; long mapping_size; @@ -2006,17 +2155,26 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, gfn_t gfn; int ret; + if (kvm_is_nested_s2_mmu(kvm, s2fd->mmu)) { + mapping = kmalloc_obj(struct kvm_guest_s2_mapping, + GFP_KERNEL_ACCOUNT); + if (!mapping) { + kvm_release_page_unused(s2vi->page); + return -ENOMEM; + } + } + kvm_fault_lock(kvm); - pgt = s2fd->vcpu->arch.hw_mmu->pgt; + pgt = s2fd->mmu->pgt; ret = -EAGAIN; if (mmu_invalidate_retry(kvm, s2vi->mmu_seq)) goto out_unlock; - perm_fault_granule = (kvm_s2_fault_is_perm(s2fd) ? - kvm_vcpu_trap_get_perm_fault_granule(s2fd->vcpu) : 0); + perm_fault_granule = kvm_s2_perm_fault_granule(s2fd); mapping_size = s2vi->vma_pagesize; pfn = s2vi->pfn; gfn = s2vi->gfn; + canonical_ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi)); /* * If we are not forced to use page mapping, check if we are @@ -2027,14 +2185,13 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, if (perm_fault_granule > PAGE_SIZE) { mapping_size = perm_fault_granule; } else { - mapping_size = transparent_hugepage_adjust(kvm, s2fd->memslot, - s2fd->hva, &pfn, - &gfn); + mapping_size = transparent_hugepage_adjust(s2fd, &pfn, &gfn); if (mapping_size < 0) { ret = mapping_size; goto out_unlock; } } + canonical_ipa = ALIGN_DOWN(canonical_ipa, mapping_size); } if (!perm_fault_granule && !s2vi->map_non_cacheable && kvm_has_mte(kvm)) @@ -2057,31 +2214,45 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(gfn), mapping_size, __pfn_to_phys(pfn), prot, memcache, flags); + /* + * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. + * We don't know which subrange is installed, track the whole + * thing. + */ + if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + kvm_record_guest_s2_mapping(pgt->mmu, canonical_ipa, + gfn_to_gpa(gfn), mapping_size, mapping); + mapping = NULL; + } } out_unlock: kvm_release_faultin_page(kvm, s2vi->page, !!ret, writable); kvm_fault_unlock(kvm); + kfree(mapping); /* * Mark the page dirty only if the fault is handled successfully, * making sure we adjust the canonical IPA if the mapping size has * been updated (via a THP upgrade, for example). */ - if (writable && !ret) { - phys_addr_t ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi)); - ipa &= ~(mapping_size - 1); - mark_page_dirty_in_slot(kvm, s2fd->memslot, gpa_to_gfn(ipa)); - } + if (writable && !ret) + mark_page_dirty_in_slot(kvm, s2fd->memslot, + gpa_to_gfn(canonical_ipa)); - if (ret != -EAGAIN) - return ret; - return 0; + if (ret == -EAGAIN) + return result ? ret : 0; + + if (result && !ret) + result->mapping_size = mapping_size; + + return ret; } -static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_result *result) { - bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); + bool perm_fault = kvm_s2_fault_is_perm(s2fd); struct kvm_s2_fault_vma_info s2vi = {}; enum kvm_pgtable_prot prot; void *memcache; @@ -2099,12 +2270,19 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) memcache = get_mmu_memcache(s2fd->vcpu); if (!perm_fault || memslot_is_logging(s2fd->memslot) || is_protected_kvm_enabled()) { - ret = topup_mmu_memcache(s2fd->vcpu, memcache); + ret = topup_mmu_memcache(s2fd->mmu, memcache); if (ret) return ret; } ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi); + if (ret == -EHWPOISON) { + /* If result is specified, let the caller handle this. */ + if (result) + return -EHWPOISON; + kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi.vma_pagesize)); + return 0; + } if (ret != 1) return ret; @@ -2114,7 +2292,19 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return ret; } - return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache); + return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache, result); +} + +static int kvm_vm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +{ + VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(s2fd) && + !esr_abt_is_write_fault(s2fd->esr) && + !kvm_s2_fault_is_exec(s2fd)); + + if (kvm_slot_has_gmem(s2fd->memslot)) + return gmem_abort(s2fd, NULL); + else + return user_mem_abort(s2fd, NULL); } /* Resolve the access fault by making the page young again. */ @@ -2224,7 +2414,9 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu) int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { struct kvm_s2_trans nested_trans, *nested = NULL; - unsigned long esr; + unsigned long esr = kvm_vcpu_get_esr(vcpu); + struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; + struct kvm *kvm = vcpu->kvm; phys_addr_t fault_ipa; /* The address we faulted on */ phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */ struct kvm_memory_slot *memslot; @@ -2233,20 +2425,18 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) gfn_t gfn; int ret, idx; - if (kvm_vcpu_abt_issea(vcpu)) + if (esr_abt_is_sea(esr)) return kvm_handle_guest_sea(vcpu); - esr = kvm_vcpu_get_esr(vcpu); - /* * The fault IPA should be reliable at this point as we're not dealing * with an SEA. */ ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); - if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm)) + if (KVM_BUG_ON(ipa == INVALID_GPA, kvm)) return -EFAULT; - is_iabt = kvm_vcpu_trap_is_iabt(vcpu); + is_iabt = esr_trap_is_iabt(esr); if (esr_fsc_is_translation_fault(esr)) { /* Beyond sanitised PARange (which is the IPA limit) */ @@ -2256,14 +2446,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } /* Falls between the IPA range and the PARange? */ - if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) { + if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(mmu->vtcr))) { fault_ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu)); return kvm_inject_sea(vcpu, is_iabt, fault_ipa); } } - trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), + trace_kvm_guest_fault(*vcpu_pc(vcpu), esr, kvm_vcpu_get_hfar(vcpu), fault_ipa); /* Check the stage-2 fault is trans. fault or write fault */ @@ -2271,14 +2461,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) !esr_fsc_is_permission_fault(esr) && !esr_fsc_is_access_flag_fault(esr) && !esr_fsc_is_excl_atomic_fault(esr)) { - kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", - kvm_vcpu_trap_get_class(vcpu), - (unsigned long)kvm_vcpu_trap_get_fault(vcpu), - (unsigned long)kvm_vcpu_get_esr(vcpu)); + kvm_err("Unsupported FSC: EC=%#lx xFSC=%#lx ESR_EL2=%#lx\n", + ESR_ELx_EC(esr), + (unsigned long)(esr & ESR_ELx_FSC), + (unsigned long)esr); return -EFAULT; } - idx = srcu_read_lock(&vcpu->kvm->srcu); + idx = srcu_read_lock(&kvm->srcu); /* * We may have faulted on a shadow stage 2 page table if we are @@ -2293,8 +2483,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * nothing to walk and we treat it as a 1:1 before going through the * canonical translation. */ - if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) && - vcpu->arch.hw_mmu->nested_stage2_enabled) { + if (kvm_is_nested_s2_mmu(kvm, mmu) && mmu->nested_stage2_enabled) { u32 esr; ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans); @@ -2321,9 +2510,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } gfn = ipa >> PAGE_SHIFT; - memslot = gfn_to_memslot(vcpu->kvm, gfn); + memslot = gfn_to_memslot(kvm, gfn); hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); - write_fault = kvm_is_write_fault(vcpu); + write_fault = esr_abt_is_write_fault(esr); if (kvm_is_error_hva(hva) || (write_fault && !writable)) { /* * The guest has put either its instructions or its page-tables @@ -2336,7 +2525,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) goto out; } - if (kvm_vcpu_abt_iss1tw(vcpu)) { + if (esr_abt_is_s1ptw(esr)) { ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); goto out_unlock; } @@ -2351,7 +2540,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * So let's assume that the guest is just being * cautious, and skip the instruction. */ - if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) { + if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) { kvm_incr_pc(vcpu); ret = 1; goto out_unlock; @@ -2369,7 +2558,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } /* Userspace should not be able to register out-of-bounds IPAs */ - VM_BUG_ON(ipa >= kvm_phys_size(vcpu->arch.hw_mmu)); + VM_BUG_ON(ipa >= kvm_phys_size(mmu)); if (esr_fsc_is_access_flag_fault(esr)) { handle_access_fault(vcpu, fault_ipa); @@ -2383,72 +2572,98 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .nested = nested, .memslot = memslot, .hva = hva, + .esr = esr, + .mmu = mmu, }; - if (kvm_vm_is_protected(vcpu->kvm)) { - ret = pkvm_mem_abort(&s2fd); - } else { - VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && - !write_fault && - !kvm_vcpu_trap_is_exec_fault(vcpu)); - - if (kvm_slot_has_gmem(memslot)) - ret = gmem_abort(&s2fd); - else - ret = user_mem_abort(&s2fd); - } - + ret = kvm->arch.vm_s2_ops->vm_mem_abort(&s2fd); if (ret == 0) ret = 1; out: if (ret == -ENOEXEC) ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu)); out_unlock: - srcu_read_unlock(&vcpu->kvm->srcu, idx); + srcu_read_unlock(&kvm->srcu, idx); return ret; } bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) + gpa_t gpa = range->start << PAGE_SHIFT; + size_t size = (range->end - range->start) << PAGE_SHIFT; + bool may_block = range->may_block; + + if (!kvm->arch.mmu.pgt) return false; - __unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, may_block); + kvm_nested_unmap_cipa_range(kvm, gpa, size, may_block); - kvm_nested_s2_unmap(kvm, range->may_block); + return false; +} + +static bool kvm_vm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, true); +} + +static bool pkvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, true); +} + +static bool no_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + /* The hypervisor doesn't support aging */ return false; } bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - u64 size = (range->end - range->start) << PAGE_SHIFT; - - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) + if (!kvm->arch.mmu.pgt) return false; - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, - range->start << PAGE_SHIFT, - size, true); + return kvm->arch.vm_s2_ops->vm_age_gfn(kvm, range); /* * TODO: Handle nested_mmu structures here using the reverse mapping in * a later version of patch series. */ } -bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +static bool kvm_vm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { u64 size = (range->end - range->start) << PAGE_SHIFT; - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) - return false; - - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT, size, false); } +static bool pkvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, false); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + if (!kvm->arch.mmu.pgt) + return false; + + return kvm->arch.vm_s2_ops->vm_test_age_gfn(kvm, range); +} + phys_addr_t kvm_mmu_get_httbr(void) { return __pa(hyp_pgtable->pgd); @@ -2699,7 +2914,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, write_lock(&kvm->mmu_lock); kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, true); - kvm_nested_s2_unmap(kvm, true); + kvm_nested_unmap_cipa_range(kvm, gpa, size, true); write_unlock(&kvm->mmu_lock); } @@ -2770,3 +2985,193 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } + +/* + * Try to walk to the specified GPA in canonical mmu - if unmapped returns 0, if + * mapped returns the granule size, otherwise returns an error. + */ +static long kvm_walk_s2(struct kvm_pgtable *pgt, + gpa_t gpa, s8 *level) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu); + kvm_pte_t pte; + long ret; + + guard(read_lock)(&kvm->mmu_lock); + + ret = kvm_pgtable_get_leaf(pgt, gpa, &pte, level, + KVM_PGTABLE_WALK_SHARED); + if (ret) + return ret; + /* Unpopulated, must fault. */ + if (!kvm_pte_valid(pte)) + return 0; + return kvm_granule_size(*level); +} + +/* Synthesised data abort at specified page table level. */ +#define PRE_FAULT_ESR(level) \ + ((ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | \ + ESR_ELx_IL | ESR_ELx_FSC_FAULT_L(level)) + +/* Retrieve either a read-only or a read/write hva. */ +static hva_t gfn_to_hva_memslot_read(struct kvm_memory_slot *slot, gfn_t gfn) +{ + return gfn_to_hva_memslot_prot(slot, gfn, /*writable=*/NULL); +} + +static long __pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu, + gpa_t gpa, struct kvm_memory_slot *memslot, s8 level) +{ + const bool is_gmem = kvm_slot_has_gmem(memslot); + const gfn_t gfn = gpa_to_gfn(gpa); + const hva_t hva = is_gmem ? 0 : gfn_to_hva_memslot_read(memslot, gfn); + const struct kvm_s2_fault_desc s2fd = { + .vcpu = vcpu, + .fault_ipa = gpa, + .nested = NULL, + .memslot = memslot, + .hva = hva, + .esr = PRE_FAULT_ESR(level), + .mmu = mmu, + }; + struct kvm_s2_fault_result result = {}; + long ret; + + if (kvm_is_error_hva(hva)) + return -EFAULT; + + if (is_gmem) + ret = gmem_abort(&s2fd, &result); + else + ret = user_mem_abort(&s2fd, &result); + if (IS_ERR_VALUE(ret)) + return ret; + return result.mapping_size; +} + +static long pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu, + gpa_t gpa, struct kvm_memory_slot *memslot) +{ + s8 level; + long ret; + + /* Try a walk first. */ + ret = kvm_walk_s2(mmu->pgt, gpa, &level); + if (ret) + return ret; + /* OK, have to fault page in. */ + return __pre_fault_s2(mmu, vcpu, gpa, memslot, level); +} + +static unsigned long +pre_fault_bytes_consumed(gpa_t gpa, unsigned long granule_size, + unsigned long bytes_remaining) +{ + /* Granules are always a power-of-2. */ + const unsigned long granule_bytes_remaining = + granule_size - (gpa % granule_size); + + return min(granule_bytes_remaining, bytes_remaining); +} + +/* If you lose the race this many times, time to give up. */ +#define MAX_PRE_FAULT_RETRIES 3 + +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (is_protected_kvm_enabled()) + return -EOPNOTSUPP; + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + return 0; +} + +/** + * kvm_arch_vcpu_pre_fault_memory - pre-fault stage-2 page tables for the + * specified GPA. + * @vcpu: The VCPU pointer + * @range: {gpa, size, flags} tuple + * + * The mapping performed is always best-effort - faulting in is necessarily + * racey. The ranges faulted in are canonical, nested page tables are ignored. + * + * @range->gpa specifies the GPA to pre-fault, @range->size specifies how many + * bytes remain to be pre-faulted and @range->flags is reserved and must be 0. + * + * Returns: the number of bytes the pre-fault consumed, or an error. + */ +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, + struct kvm_pre_fault_memory *range) +{ + struct kvm *kvm = vcpu->kvm; + const u64 bytes_remaining = range->size; + struct kvm_s2_mmu *mmu = &kvm->arch.mmu; /* Canonical. */ + struct kvm_memory_slot *memslot; + const gpa_t gpa = range->gpa; + int num_retries = 0; + long ret; + + memslot = gfn_to_memslot(kvm, gpa_to_gfn(gpa)); + if (!memslot) + return -ENOENT; + /* SRCU must be released for progress and only userland can do that. */ + if (memslot->flags & KVM_MEMSLOT_INVALID) + return -EAGAIN; + + do { + ret = pre_fault_s2(mmu, vcpu, gpa, memslot); + } while (ret == -EAGAIN && num_retries++ < MAX_PRE_FAULT_RETRIES); + + if (IS_ERR_VALUE(ret)) + return ret; + return pre_fault_bytes_consumed(gpa, ret, bytes_remaining); +} + +static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = no_age_gfn, + .vm_test_age_gfn = no_age_gfn, + .vm_stage2_unmap_range = no_stage2_unmap_range, + .vm_mem_abort = protected_pkvm_mem_abort, +}; + +static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = pkvm_age_gfn, + .vm_test_age_gfn = pkvm_test_age_gfn, + .vm_stage2_unmap_range = pkvm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, +}; + +static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { + .vm_flush_remote_tlbs = kvm_vm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = kvm_vm_flush_remote_tlbs_range, + .vm_age_gfn = kvm_vm_age_gfn, + .vm_test_age_gfn = kvm_vm_test_age_gfn, + .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, +}; + +#define KVM_VM_S2_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vm_s2_ops *arm64_vm_s2_ops[] = { + KVM_VM_S2_OPS(VM_VHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_NVHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_PKVM, pkvm_vm_s2_ops), + KVM_VM_S2_OPS(VM_PROTECTED_PKVM, protected_pkvm_vm_s2_ops), +}; + +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vm_s2_ops) != VM_FLAVOR_MAX); + + kvm->arch.vm_s2_ops = arm64_vm_s2_ops[kvm->arch.vm_flavor]; + if (WARN_ON(!kvm->arch.vm_s2_ops)) + return -EINVAL; + return 0; +}
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index b191365..f1c414f 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c
@@ -5,6 +5,7 @@ */ #include <linux/bitfield.h> +#include <linux/interval_tree.h> #include <linux/kvm.h> #include <linux/kvm_host.h> @@ -44,14 +45,16 @@ struct vncr_tlb { * will invalidate them more often). */ #define S2_MMU_PER_VCPU 2 +#define S2_MMU_PER_VM (KVM_MAX_VCPUS * S2_MMU_PER_VCPU) int kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, - KVM_MAX_VCPUS * S2_MMU_PER_VCPU, + S2_MMU_PER_VM, GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; atomic_set(&kvm->arch.vncr_tlb_count, 0); + spin_lock_init(&kvm->arch.guest_s2_tracking_lock); return kvm->arch.nested_mmus ? 0 : -ENOMEM; } @@ -124,10 +127,15 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) return ret; } + for (i = 0; i < S2_MMU_PER_VCPU; i++) + kvm_nested_s2_ptdump_create_debugfs(&tmp[i], i + kvm->arch.nested_mmus_size); + guard(write_lock)(&kvm->mmu_lock); - for (i = 0; i < S2_MMU_PER_VCPU; i++) + for (i = 0; i < S2_MMU_PER_VCPU; i++) { + tmp[i].s2_mmu_idx = i + kvm->arch.nested_mmus_size; kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; + } kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; } @@ -657,7 +665,7 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) return 0; tmp &= ~(sz - 1); - if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) + if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0)) goto again; if (!(pte & PTE_VALID)) goto again; @@ -837,10 +845,8 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; /* Make sure we don't forget to do the laundry */ - if (kvm_s2_mmu_valid(s2_mmu)) { - kvm_nested_s2_ptdump_remove_debugfs(s2_mmu); + if (kvm_s2_mmu_valid(s2_mmu)) s2_mmu->pending_unmap = true; - } /* * The virtual VMID (modulo CnP) will be used as a key when matching @@ -854,8 +860,6 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; - kvm_nested_s2_ptdump_create_debugfs(s2_mmu); - out: atomic_inc(&s2_mmu->refcnt); @@ -871,6 +875,87 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) return s2_mmu; } +#define S2_MMU_IDX_MASK GENMASK_ULL(11, 0) + +static void tag_s2_mapping_mmu(struct kvm_guest_s2_mapping *mapping, + struct kvm_s2_mmu *mmu) +{ + BUILD_BUG_ON(S2_MMU_PER_VM > SZ_4K); + mapping->nested.start &= ~S2_MMU_IDX_MASK; + mapping->nested.start |= mmu->s2_mmu_idx; +} + +static struct kvm_s2_mmu *s2_mapping_to_mmu(struct kvm *kvm, + struct kvm_guest_s2_mapping *mapping) +{ + return kvm->arch.nested_mmus[mapping->nested.start & S2_MMU_IDX_MASK]; +} + +static unsigned long s2_mapping_to_nested_start(struct kvm_guest_s2_mapping *mapping) +{ + return mapping->nested.start & ~S2_MMU_IDX_MASK; +} + +void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, + gpa_t nested_ipa, size_t map_size, + struct kvm_guest_s2_mapping *mapping) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + lockdep_assert_held_read(&kvm->mmu_lock); + + canonical_ipa = ALIGN_DOWN(canonical_ipa, map_size); + nested_ipa = ALIGN_DOWN(nested_ipa, map_size); + + mapping->canonical.start = canonical_ipa; + mapping->canonical.last = canonical_ipa + map_size - 1; + + mapping->nested.start = nested_ipa; + mapping->nested.last = nested_ipa + map_size - 1; + + tag_s2_mapping_mmu(mapping, mmu); + + guard(spinlock)(&kvm->arch.guest_s2_tracking_lock); + interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings); + interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings); +} + +void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, + size_t size) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + struct interval_tree_node *node, *next; + struct kvm_guest_s2_mapping *mapping; + gpa_t nipa_end = nipa + size - 1; + + /* + * See kvm_nested_unmap_cipa_range() for why guest_s2_tracking_lock + * isn't taken here. + */ + lockdep_assert_held_write(&kvm->mmu_lock); + + node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end); + while (node) { + unsigned long nested_start; + + next = interval_tree_iter_next(node, nipa, nipa_end); + mapping = container_of(node, struct kvm_guest_s2_mapping, + nested); + /* + * Tracking must be conservative on removal, only remove + * mappings that are within the unmap range. + */ + nested_start = s2_mapping_to_nested_start(mapping); + if (nipa <= nested_start && nipa_end >= mapping->nested.last) { + interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); + interval_tree_remove(&mapping->canonical, + &kvm->arch.mmu.guest_s2_mappings); + kfree(mapping); + } + node = next; + } +} + void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) { /* CnP being set denotes an invalid entry */ @@ -1293,6 +1378,56 @@ void kvm_nested_s2_wp(struct kvm *kvm) kvm_invalidate_vncr_ipa_all(kvm); } +void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size, + bool may_block) +{ + gpa_t cipa_end = cipa + unmap_size - 1; + struct kvm_guest_s2_mapping *mapping; + struct interval_tree_node *node; + size_t mapping_size; + + /* + * Guest s2 tracking interval trees are only accessed while holding the + * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the + * mmu_lock is held for write. This saves us from having to manually + * lock/unlock guest_s2_tracking_lock below around + * cond_resched_rwlock_write(). + */ + lockdep_assert_held_write(&kvm->mmu_lock); + + if (!kvm->arch.nested_mmus_size) + return; + + while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings, + cipa, cipa_end))) { + unsigned long nested_start; + struct kvm_s2_mmu *mmu; + + mapping = container_of(node, struct kvm_guest_s2_mapping, + canonical); + nested_start = s2_mapping_to_nested_start(mapping); + mmu = s2_mapping_to_mmu(kvm, mapping); + + /* We could race against MMU teardown, which frees mmu->pgt. */ + if (mmu->pgt) { + mapping_size = mapping->nested.last - nested_start + 1; + + if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mmu->pgt, nested_start, + mapping_size))) + return; + + interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); + } + interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings); + kfree(mapping); + + if (may_block) + cond_resched_rwlock_write(&kvm->mmu_lock); + } + + kvm_invalidate_vncr_ipa(kvm, cipa, cipa + unmap_size); +} + void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) { int i; @@ -1424,7 +1559,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) bool write_fault, writable; unsigned long mmu_seq; struct vncr_tlb *vt; - struct page *page; + struct page *page = NULL; u64 va, pfn, gfn; int ret; @@ -1480,7 +1615,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) return -EFAULT; } } else { - ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL); if (ret) { kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, write_fault, false, false);
diff --git a/arch/arm64/kvm/pauth.c b/arch/arm64/kvm/pauth.c index d5eb3ae..bb6c42e 100644 --- a/arch/arm64/kvm/pauth.c +++ b/arch/arm64/kvm/pauth.c
@@ -5,7 +5,7 @@ * * Primitive PAuth emulation for ERETAA/ERETAB. * - * This code assumes that is is run from EL2, and that it is part of + * This code assumes that it is run from EL2, and that it is part of * the emulation of ERETAx for a guest hypervisor. That's a lot of * baked-in assumptions and shortcuts. *
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4..380dfb1 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c
@@ -16,6 +16,9 @@ #include "hyp_constants.h" +#define CREATE_TRACE_POINTS +#include "trace_pkvm.h" + DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory); @@ -85,6 +88,77 @@ void __init kvm_hyp_reserve(void) hyp_mem_base); } +static int pkvm_hyp_topup(enum pkvm_topup_id id, unsigned long nr_pages) +{ + struct kvm_hyp_memcache mc; + struct arm_smccc_res res; + int ret; + + init_hyp_memcache(&mc); + ret = topup_hyp_memcache(&mc, nr_pages); + if (ret) + goto err; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_topup), id, mc.head, + mc.nr_pages, &res); + if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS)) { + ret = -EINVAL; + goto err; + } + + ret = res.a1; + mc.head = res.a2; + mc.nr_pages = res.a3; + +err: + free_hyp_memcache(&mc); + return ret; +} + +static unsigned long __pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +{ + struct kvm_hyp_memcache mc; + struct arm_smccc_res res; + unsigned long reclaimed; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_reclaim), id, target, &res); + if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS) || WARN_ON_ONCE(res.a1)) + return 0; + + init_hyp_memcache(&mc); + mc.head = res.a2; + mc.nr_pages = reclaimed = res.a3; + free_hyp_memcache(&mc); + + return reclaimed; +} + +static unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +{ + unsigned long reclaimed = 0; + + while (reclaimed < target) { + /* Arbitrary limit to avoid blocking in EL2 for too long */ + unsigned long r = __pkvm_hyp_reclaim(id, min(target - reclaimed, 16)); + + if (!r) + break; + + reclaimed += r; + if (reclaimed >= target) + break; + + cond_resched(); + } + + return reclaimed; +} + +static unsigned long pkvm_hyp_reclaimable(enum pkvm_topup_id id) +{ + return kvm_call_hyp_nvhe(__pkvm_hyp_reclaimable, id); +} + static void __pkvm_destroy_hyp_vm(struct kvm *kvm) { if (pkvm_hyp_vm_is_created(kvm)) { @@ -100,30 +174,34 @@ static void __pkvm_destroy_hyp_vm(struct kvm *kvm) kvm->arch.pkvm.handle = 0; kvm->arch.pkvm.is_created = false; - free_hyp_memcache(&kvm->arch.pkvm.teardown_mc); free_hyp_memcache(&kvm->arch.pkvm.stage2_teardown_mc); } static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) { - size_t hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE); pkvm_handle_t handle = vcpu->kvm->arch.pkvm.handle; - void *hyp_vcpu; int ret; - vcpu->arch.pkvm_memcache.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; + init_hyp_stage2_memcache(&vcpu->arch.stage2_mc); - hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT); - if (!hyp_vcpu) - return -ENOMEM; + ret = pkvm_call_hyp_req(__pkvm_init_vcpu, handle, vcpu); + if (ret) + return ret; - ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, vcpu, hyp_vcpu); - if (!ret) - vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); - else - free_pages_exact(hyp_vcpu, hyp_vcpu_sz); + /* + * Mirror EL2's seeding of power_state from mp_state. The hyp vCPU is + * published, so take mp_state_lock against kvm_psci_vcpu_on(). + */ + if (vcpu_is_protected(vcpu)) { + spin_lock(&vcpu->arch.mp_state_lock); + if (kvm_arm_vcpu_stopped(vcpu)) + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_UNINITIALIZED); + spin_unlock(&vcpu->arch.mp_state_lock); + } - return ret; + vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); + + return 0; } /* @@ -138,8 +216,8 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) */ static int __pkvm_create_hyp_vm(struct kvm *kvm) { - size_t pgd_sz, hyp_vm_sz; - void *pgd, *hyp_vm; + size_t pgd_sz; + void *pgd; int ret; if (kvm->created_vcpus < 1) @@ -156,28 +234,15 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm) if (!pgd) return -ENOMEM; - /* Allocate memory to donate to hyp for vm and vcpu pointers. */ - hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE, - size_mul(sizeof(void *), - kvm->created_vcpus))); - hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT); - if (!hyp_vm) { - ret = -ENOMEM; - goto free_pgd; - } - - /* Donate the VM memory to hyp and let hyp initialize it. */ - ret = kvm_call_hyp_nvhe(__pkvm_init_vm, kvm, hyp_vm, pgd); + ret = pkvm_call_hyp_req(__pkvm_init_vm, kvm, pgd); if (ret) - goto free_vm; + goto free_pgd; kvm->arch.pkvm.is_created = true; - kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; + init_hyp_stage2_memcache(&kvm->arch.pkvm.stage2_teardown_mc); kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE); return 0; -free_vm: - free_pages_exact(hyp_vm, hyp_vm_sz); free_pgd: free_pages_exact(pgd, pgd_sz); return ret; @@ -229,10 +294,9 @@ void pkvm_destroy_hyp_vm(struct kvm *kvm) mutex_unlock(&kvm->arch.config_lock); } -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) +int pkvm_init_host_vm(struct kvm *kvm) { int ret; - bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); @@ -240,8 +304,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) return ret; kvm->arch.pkvm.handle = ret; - kvm->arch.pkvm.is_protected = protected; - if (protected) { + if (kvm_vm_is_protected(kvm)) { pr_warn_once("kvm: protected VMs are experimental and for development only, tainting kernel\n"); add_taint(TAINT_USER, LOCKDEP_STILL_OK); } @@ -270,8 +333,55 @@ static int __init pkvm_drop_host_privileges(void) return ret; } +void __init pkvm_selftests(void) +{ +#ifdef CONFIG_NVHE_EL2_DEBUG + int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest); + unsigned long reclaimed; + + reclaimed = pkvm_hyp_reclaim(PKVM_TOPUP_HYP_ALLOC_SELFTEST, ULONG_MAX); + + /* On failure, not all the pages may be reclaimable */ + if (!ret) + WARN_ON(reclaimed != 6 /* SELFTEST_MAX_PAGES */); + else + kvm_err("pKVM hyp allocator selftest failed (%d)\n", ret); +#endif +} + +static unsigned long pkvm_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long reclaimable = 0; + int id; + + for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) + reclaimable += pkvm_hyp_reclaimable(id); + + return reclaimable ?: SHRINK_EMPTY; +} + +static unsigned long pkvm_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long reclaimed = 0; + int id; + + sc->nr_scanned = 0; + + for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) { + unsigned long r = pkvm_hyp_reclaim(id, sc->nr_to_scan - sc->nr_scanned); + + reclaimed += r; + sc->nr_scanned += r; + if (sc->nr_scanned >= sc->nr_to_scan) + break; + } + + return reclaimed ?: SHRINK_STOP; +} + static int __init finalize_pkvm(void) { + struct shrinker *pkvm_shrinker; int ret; if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised()) @@ -287,10 +397,21 @@ static int __init finalize_pkvm(void) kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size); ret = pkvm_drop_host_privileges(); - if (ret) + if (ret) { pr_err("Failed to finalize Hyp protection: %d\n", ret); + return ret; + } - return ret; + pkvm_shrinker = shrinker_alloc(0, "pkvm"); + if (pkvm_shrinker) { + pkvm_shrinker->count_objects = pkvm_shrinker_count; + pkvm_shrinker->scan_objects = pkvm_shrinker_scan; + shrinker_register(pkvm_shrinker); + } else { + kvm_err("Failed to register shrinker for pKVM\n"); + } + + return 0; } device_initcall_sync(finalize_pkvm); @@ -414,6 +535,9 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 end = addr + size; int ret; + if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size))) + return -EINVAL; + lockdep_assert_held_write(&kvm->mmu_lock); mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1); @@ -594,3 +718,31 @@ bool pkvm_force_reclaim_guest_page(phys_addr_t phys) return !ret || ret == -EAGAIN; } + +static int pkvm_handle_hyp_req(struct pkvm_hyp_req *req) +{ + int ret = -EINVAL; + + switch (req->type) { + case PKVM_HYP_REQ_HYP_ALLOC: + ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC, req->mem.nr_pages); + break; + case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: + ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC_SELFTEST, req->mem.nr_pages); + break; + } + + trace_kvm_handle_pkvm_hyp_req(req, ret); + + return ret; +} + +int __pkvm_handle_smccc_req(struct arm_smccc_res *res) +{ + struct pkvm_hyp_req req; + + if (smccc_to_pkvm_hyp_req(&req, res)) + return pkvm_handle_hyp_req(&req); + + return res->a1; +}
diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 5b1af7e..e2b28db 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c
@@ -50,8 +50,11 @@ static u32 __kvm_pmu_event_mask(unsigned int pmuver) case ID_AA64DFR0_EL1_PMUVer_V3P5: case ID_AA64DFR0_EL1_PMUVer_V3P7: return GENMASK(15, 0); - default: /* Shouldn't be here, just for sanity */ + default: + /* Shouldn't be here, just for sanity */ WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); + fallthrough; + case ID_AA64DFR0_EL1_PMUVer_NI: return 0; } }
diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index e3db844..b01f8d4 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c
@@ -63,7 +63,12 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) return PSCI_RET_INVALID_PARAMS; spin_lock(&vcpu->arch.mp_state_lock); - if (!kvm_arm_vcpu_stopped(vcpu)) { + /* + * A protected vCPU is off once the host has acted on EL2's CPU_OFF. + * STOPPED before that is the VMM's pause. + */ + if (!kvm_arm_vcpu_stopped(vcpu) || + (vcpu_is_protected(vcpu) && !kvm_pkvm_vcpu_is_powered_off(vcpu))) { if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) ret = PSCI_RET_ALREADY_ON; else @@ -161,7 +166,9 @@ static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) */ kvm_for_each_vcpu(i, tmp, vcpu->kvm) { spin_lock(&tmp->arch.mp_state_lock); - WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + /* Keep EL2's off record for the guest's next CPU_ON. */ + if (!kvm_pkvm_vcpu_is_powered_off(tmp)) + WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); spin_unlock(&tmp->arch.mp_state_lock); } kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
diff --git a/arch/arm64/kvm/ptdump.c b/arch/arm64/kvm/ptdump.c index 6989979..9f0aeba 100644 --- a/arch/arm64/kvm/ptdump.c +++ b/arch/arm64/kvm/ptdump.c
@@ -17,7 +17,7 @@ #define MARKERS_LEN 2 #define KVM_PGTABLE_MAX_LEVELS (KVM_PGTABLE_LAST_LEVEL + 1) -#define S2FNAMESZ sizeof("0x0123456789abcdef-0x0123456789abcdef-s2-disabled") +#define S2FNAMESZ sizeof("nested_mmu0000") struct kvm_ptdump_guest_state { struct kvm_s2_mmu *mmu; @@ -115,15 +115,28 @@ static int kvm_ptdump_build_levels(struct ptdump_pg_level *level, u32 start_lvl) static struct kvm_ptdump_guest_state *kvm_ptdump_parser_create(struct kvm_s2_mmu *mmu) { + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_ptdump_guest_state *st; - struct kvm_pgtable *pgtable = mmu->pgt; + s8 start_level; int ret; + /* + * We only need the pgt start level to initialize the ptdump, get it + * while holding the mmu_lock. It's fine if the pgt gets freed + * afterwards, we'll check again when doing the actual dump. + */ + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu->pgt) + start_level = mmu->pgt->start_level; + else + return ERR_PTR(-ENOENT); + } + st = kzalloc_obj(struct kvm_ptdump_guest_state, GFP_KERNEL_ACCOUNT); if (!st) return ERR_PTR(-ENOMEM); - ret = kvm_ptdump_build_levels(&st->level[0], pgtable->start_level); + ret = kvm_ptdump_build_levels(&st->level[0], start_level); if (ret) { kfree(st); return ERR_PTR(ret); @@ -149,6 +162,9 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused) }; guard(write_lock)(&kvm->mmu_lock); + if (!mmu->pgt) + return 0; + st->parser_state = (struct ptdump_pg_state) { .marker = &st->ipa_marker[0], .end_address = BIT(mmu->pgt->ia_bits), @@ -157,6 +173,15 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused) .seq = m, }; + if (kvm_is_nested_s2_mmu(kvm, mmu)) { + if (kvm_s2_mmu_valid(mmu)) + seq_printf(m, "VTCR: 0x%016llx VTTBR: 0x%016llx s2: %s\n", + mmu->tlb_vtcr, mmu->tlb_vttbr, + mmu->nested_stage2_enabled ? "enabled" : "disabled"); + else + return 0; + } + ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker); if (ret) return ret; @@ -211,17 +236,27 @@ static const struct file_operations kvm_ptdump_guest_fops = { static int kvm_pgtable_range_show(struct seq_file *m, void *unused) { - struct kvm_pgtable *pgtable = m->private; + struct kvm_s2_mmu *mmu = m->private; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - seq_printf(m, "%2u\n", pgtable->ia_bits); + guard(read_lock)(&kvm->mmu_lock); + + if (mmu->pgt) + seq_printf(m, "%2u\n", mmu->pgt->ia_bits); + return 0; } static int kvm_pgtable_levels_show(struct seq_file *m, void *unused) { - struct kvm_pgtable *pgtable = m->private; + struct kvm_s2_mmu *mmu = m->private; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - pgtable->start_level); + guard(read_lock)(&kvm->mmu_lock); + + if (mmu->pgt) + seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - mmu->pgt->start_level); + return 0; } @@ -230,15 +265,12 @@ static int kvm_pgtable_debugfs_open(struct inode *m, struct file *file, { struct kvm_s2_mmu *mmu = m->i_private; struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - struct kvm_pgtable *pgtable; int ret; if (!kvm_get_kvm_safe(kvm)) return -ENOENT; - pgtable = mmu->pgt; - - ret = single_open(file, show, pgtable); + ret = single_open(file, show, mmu); if (ret < 0) kvm_put_kvm(kvm); return ret; @@ -276,30 +308,23 @@ static const struct file_operations kvm_pgtable_levels_fops = { .release = kvm_pgtable_debugfs_close, }; -void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) +void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) { - struct dentry *dent; char file_name[S2FNAMESZ]; - snprintf(file_name, sizeof(file_name), "0x%016llx-0x%016llx-s2-%sabled", - mmu->tlb_vttbr, - mmu->tlb_vtcr, - mmu->nested_stage2_enabled ? "en" : "dis"); + snprintf(file_name, sizeof(file_name), "nested_mmu%d", idx); - dent = debugfs_create_file(file_name, 0400, - mmu->arch->debugfs_nv_dentry, mmu, - &kvm_ptdump_guest_fops); - - mmu->shadow_pt_debugfs_dentry = dent; -} - -void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) -{ - debugfs_remove(mmu->shadow_pt_debugfs_dentry); + debugfs_create_file(file_name, 0400, + mmu->arch->debugfs_nv_dentry, mmu, + &kvm_ptdump_guest_fops); } void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) { + /* pKVM handles the page-tables in the hypervisor */ + if (is_protected_kvm_enabled()) + return; + debugfs_create_file("stage2_page_tables", 0400, kvm->debugfs_dentry, &kvm->arch.mmu, &kvm_ptdump_guest_fops); debugfs_create_file("ipa_range", 0400, kvm->debugfs_dentry,
diff --git a/arch/arm64/kvm/pvtime.c b/arch/arm64/kvm/pvtime.c index 4ceabaa..053fe83 100644 --- a/arch/arm64/kvm/pvtime.c +++ b/arch/arm64/kvm/pvtime.c
@@ -67,9 +67,9 @@ gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu) return base; } -bool kvm_arm_pvtime_supported(void) +bool kvm_arm_pvtime_supported(struct kvm *kvm) { - return !!sched_info_on(); + return !!sched_info_on() && (!kvm || !kvm_vm_is_protected(kvm)); } int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, @@ -81,7 +81,7 @@ int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, int ret = 0; int idx; - if (!kvm_arm_pvtime_supported() || + if (!kvm_arm_pvtime_supported(kvm) || attr->attr != KVM_ARM_VCPU_PVTIME_IPA) return -ENXIO; @@ -110,7 +110,7 @@ int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu, u64 __user *user = (u64 __user *)attr->addr; u64 ipa; - if (!kvm_arm_pvtime_supported() || + if (!kvm_arm_pvtime_supported(vcpu->kvm) || attr->attr != KVM_ARM_VCPU_PVTIME_IPA) return -ENXIO; @@ -126,7 +126,7 @@ int kvm_arm_pvtime_has_attr(struct kvm_vcpu *vcpu, { switch (attr->attr) { case KVM_ARM_VCPU_PVTIME_IPA: - if (kvm_arm_pvtime_supported()) + if (kvm_arm_pvtime_supported(vcpu->kvm)) return 0; } return -ENXIO;
diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c index 9724c32..69377195 100644 --- a/arch/arm64/kvm/stacktrace.c +++ b/arch/arm64/kvm/stacktrace.c
@@ -198,9 +198,6 @@ static void hyp_dump_backtrace(unsigned long hyp_offset) } #ifdef CONFIG_PKVM_STACKTRACE -DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], - pkvm_stacktrace); - /* * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace. *
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5e6b04c..edfc3be 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c
@@ -1029,7 +1029,7 @@ static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu, static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - if (kvm_vcpu_has_pmu(vcpu)) + if (kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) return 0; return REG_HIDDEN; @@ -1059,10 +1059,6 @@ static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - /* This thing will UNDEF, who cares about the reset value? */ - if (!kvm_vcpu_has_pmu(vcpu)) - return 0; - reset_unknown(vcpu, r); __vcpu_rmw_sys_reg(vcpu, r->reg, &=, kvm_pmu_evtyper_mask(vcpu->kvm)); @@ -2514,9 +2510,10 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, /* * Once the VM has started the ID registers are immutable. Reject any - * write that does not match the final register value. + * write that does not match the final register value once we have + * got far enough into first running the VM to use the values. */ - if (kvm_vm_has_ran_once(vcpu->kvm)) { + if (kvm_id_regs_final(vcpu->kvm)) { if (val != read_id_reg(vcpu, rd)) ret = -EBUSY; else @@ -2550,7 +2547,7 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val) lockdep_assert_held(&kvm->arch.config_lock); - if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm)) + if (KVM_BUG_ON(kvm_id_regs_final(kvm) || !p, kvm)) return; *p = val; @@ -3246,10 +3243,10 @@ static int set_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, return -EINVAL; /* - * Once the VM has started the ID registers are immutable. Reject the - * write if userspace tries to change it. + * Once we have been far enough into starting the VM the ID registers + * are immutable. Reject the write if userspace tries to change it. */ - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return -EBUSY; /* @@ -5864,26 +5861,17 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu) } /* - * Perform last adjustments to the ID registers that are implied by the + * Do system register finalization that is shared by the whole guest. This + * includes last adjustments to the ID registers that are implied by the * configuration outside of the ID regs themselves, as well as any * initialisation that directly depend on these ID registers (such as * RES0/RES1 behaviours). This is not the place to configure traps though. - * - * Because this can be called once per CPU, changes must be idempotent. */ -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) +static int kvm_vm_finalize_sys_regs(struct kvm *kvm) { - struct kvm *kvm = vcpu->kvm; + lockdep_assert_held(&kvm->arch.config_lock); - guard(mutex)(&kvm->arch.config_lock); - - if (vcpu_has_nv(vcpu)) { - int ret = kvm_init_nv_sysregs(vcpu); - if (ret) - return ret; - } - - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return 0; /* @@ -5931,12 +5919,37 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) kvm_vgic_finalize_idregs(kvm); } + set_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &kvm->arch.flags); + + return 0; +} + +/* + * Because this can be called once per CPU, changes must be idempotent. + */ +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + int ret; + + guard(mutex)(&kvm->arch.config_lock); + + ret = kvm_vm_finalize_sys_regs(kvm); + if (ret) + return ret; + + if (vcpu_has_nv(vcpu)) { + ret = kvm_init_nv_sysregs(vcpu); + if (ret) + return ret; + } + return 0; } int __init kvm_sys_reg_table_init(void) { - const struct sys_reg_desc *gicv3_regs; + const struct sys_reg_desc *gicv3_regs, *gicv5_regs; bool valid = true; unsigned int i, sz; int ret = 0; @@ -5949,8 +5962,12 @@ int __init kvm_sys_reg_table_init(void) valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false); valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false); + /* The GICv3 system registers... */ gicv3_regs = vgic_v3_get_sysreg_table(&sz); valid &= check_sysreg_table(gicv3_regs, sz, false); + /* ...and the GICv5 system registers. */ + gicv5_regs = vgic_v5_get_sysreg_table(&sz); + valid &= check_sysreg_table(gicv5_regs, sz, false); if (!valid) return -EINVAL;
diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index bd56a45..a3cccad 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h
@@ -254,7 +254,7 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index); -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu); +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu); #define AA32(_x) .aarch32_map = AA32_##_x #define Op0(_x) .Op0 = _x
diff --git a/arch/arm64/kvm/trace_pkvm.h b/arch/arm64/kvm/trace_pkvm.h new file mode 100644 index 0000000..493225e --- /dev/null +++ b/arch/arm64/kvm/trace_pkvm.h
@@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(_TRACE_PKVM_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PKVM_ARM64_KVM_H + +#include <linux/tracepoint.h> +#include <asm/kvm_pkvm.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +TRACE_DEFINE_ENUM(PKVM_HYP_NO_REQ); +TRACE_DEFINE_ENUM(PKVM_HYP_REQ_HYP_ALLOC); + +#define PKVM_HYP_REQ_TYPES \ + { PKVM_HYP_NO_REQ, "NO_REQ" }, \ + { PKVM_HYP_REQ_HYP_ALLOC, "HYP_ALLOC" } + +TRACE_EVENT(kvm_handle_pkvm_hyp_req, + TP_PROTO(struct pkvm_hyp_req *req, int ret), + TP_ARGS(req, ret), + + TP_STRUCT__entry( + __field(u8, type) + __field(int, ret) + ), + + TP_fast_assign( + __entry->type = req->type; + __entry->ret = ret; + ), + + TP_printk("type: %s ret: %d", + __print_symbolic(__entry->type, PKVM_HYP_REQ_TYPES), + __entry->ret) +); + +#endif /* _TRACE_PKVM_ARM64_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_pkvm + +/* This part must be outside protection */ +#include <trace/define_trace.h>
diff --git a/arch/arm64/kvm/vgic-sys-reg-v5.c b/arch/arm64/kvm/vgic-sys-reg-v5.c new file mode 100644 index 0000000..6b255ea --- /dev/null +++ b/arch/arm64/kvm/vgic-sys-reg-v5.c
@@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +/* + * VGICv5 system registers handling functions for AArch64 mode + */ + +#include <linux/irqchip/arm-gic-v5.h> + +#include <linux/kvm.h> +#include <linux/kvm_host.h> +#include <linux/wordpart.h> + +#include <asm/kvm_emulate.h> + +#include "vgic/vgic.h" +#include "sys_regs.h" + +#define ICC_PPI_PRIORITYR_PRIORITY_MASK REPEAT_BYTE(0x1f) + +static int set_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* The upper 32 bits are RES0 */ + cpu_if->vgic_apr = val & ~ICC_APR_EL1_RES0; + + return 0; +} + +static int get_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = cpu_if->vgic_apr; + + return 0; +} + +static int set_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * We only support setting the ICC_CR0_EL1.En bit, which is actually + * stored in the VMCR. + */ + FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_EN, &cpu_if->vgic_vmcr, + FIELD_GET(ICC_CR0_EL1_EN, val)); + + return 0; +} + +static int get_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * PID only applies if EL3 is present. Same applies to IPPT. Hence, + * those fields are always presented as 0. + * + * We always present the link as connected and idle: + * (LINK = 1, LINK_IDLE = 1). + */ + *val = FIELD_PREP(ICC_CR0_EL1_EN, + FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr)); + *val |= ICC_CR0_EL1_LINK_MASK; + *val |= ICC_CR0_EL1_LINK_IDLE_MASK; + + return 0; +} + +static int set_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* Set the VPMR field in the VMCR */ + FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_VPMR, &cpu_if->vgic_vmcr, + FIELD_GET(ICC_PCR_EL1_PRIORITY, val)); + + return 0; +} + +static int get_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = FIELD_PREP(ICC_PCR_EL1_PRIORITY, + FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr)); + + return 0; +} + +static int set_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + cpu_if->vgic_icsr = val & ~ICC_ICSR_EL1_RES0; + + return 0; +} + +static int get_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = cpu_if->vgic_icsr; + + return 0; +} + +/* + * Helper macro to iterate over a range of PPIs and execute some code (to either + * extract or set the vgic_irq state). This is used when `get`-ing the PPI + * ENABLER, ACTIVER, PENDR and when setting the PRIORITYR state. + * + * vcpu: Pointer to struct kvm_vcpu (to which these PPIs belong) + * r: The register index. 0 or 1 for all except PRIORITYR (which is 0-15) + * nr: The number of PPIs iterated over. 64 for all but PRIORITYR (which is 8) + * code: The code snippet to execute for each vgic_irq + */ +#define for_ppi_state(vcpu, r, nr, code) \ + do { \ + struct kvm_vcpu *__vcpu = (vcpu); \ + int __r = (r); \ + int __nr = (nr); \ + \ + for (int i = 0; i < __nr; i++) { \ + u32 id = vgic_v5_make_ppi(__r * __nr + i); \ + struct vgic_irq *irq; \ + \ + irq = vgic_get_vcpu_irq(__vcpu, id); \ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { \ + code; \ + } \ + vgic_put_irq(__vcpu->kvm, irq); \ + } \ + } while (0) + +static int set_gic_ppi_enabler(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int i, start, end, reg = r->Op2 % 2; + + /* + * If we're only handling architected PPIs and the guest writes to the + * enable for the non-architected PPIs, we just return as there's + * nothing to do at all. We don't even allocate the storage for them in + * this case. + */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Merge the raw guest write into our bitmap at an offset of either 0 or + * 64. + * + * Note that there is *NO* masking applied - the enable state is written + * unfiltered. The assumption is that userspace uses this interface to + * set initial state before the guest runs, and then the exposed PPI + * mask is applied later, when vgic_v5_finalize_ppi_state() runs on + * first entry to each vCPU. If userspace chooses to set the enabler + * state later, it is fully capable of breaking the illusion we provided + * to the guest by exposing register state (and PPIs) to the guest that + * were not initially exposed. Good luck! + */ + bitmap_write(cpu_if->vgic_ppi_enabler, val, 64 * reg, 64); + + /* + * Sync the change in enable states to the vgic_irqs for the written + * register slice. + */ + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler); + + vgic_put_irq(vcpu->kvm, irq); + } + + return 0; +} + +static int get_gic_ppi_enabler(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long enabler = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the ENABLER value. */ + for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &enabler, irq->enabled)); + + *val = enabler; + + return 0; +} + +static int set_gic_ppi_activer(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int i, start, end, reg = r->Op2 % 2; + + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Store the raw guest write. The exposed PPI mask is applied later, + * when vgic_v5_finalize_ppi_state() runs on first entry to each + * vCPU. See comment on set_gic_ppi_enabler() for details. + */ + bitmap_write(cpu_if->vgic_ppi_activer, val, 64 * reg, 64); + + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->active = test_bit(i, cpu_if->vgic_ppi_activer); + + vgic_put_irq(vcpu->kvm, irq); + } + + return 0; +} + +static int get_gic_ppi_activer(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long activer = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the ACTIVER value. */ + for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &activer, irq->active)); + + *val = activer; + + return 0; +} + +static int set_gic_ppi_pendr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + int i, start, end, reg = r->Op2 % 2; + + /* If we only support architected PPIs, return */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Update each struct vgic_irq with the pending state, treating Level + * and Edge interrupts differently. The exposed PPI mask is applied + * later, when vgic_v5_finalize_ppi_state() runs on first entry to each + * vCPU. See comment on set_gic_ppi_enabler() for details. + */ + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + bool level = !!(val & BIT_ULL(i - start)); + + if (irq->config == VGIC_CONFIG_LEVEL) + irq->line_level = level; + else + irq->pending_latch = level; + } + + vgic_put_irq(vcpu->kvm, irq); + } + + /* + * The pending state is generated from the vgic_irqs on each guest + * entry. Therefore, we don't store the raw value written anywhere in + * the case of userspace PPI_PENDRx_EL1 writes. + */ + + return 0; +} + +static int get_gic_ppi_pendr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long pendr = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the PENDR value. */ + for_ppi_state(vcpu, reg, 64, { + if (irq_is_pending(irq)) + __assign_bit(i % 64, &pendr, 1); + }); + + *val = pendr; + + return 0; +} + +static int set_gic_ppi_priorityr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int reg = ((r->CRm & 0x1) << 3) + r->Op2; + + /* If we only support architected PPIs, return */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) + return 0; + + val &= ICC_PPI_PRIORITYR_PRIORITY_MASK; + + /* + * Although priorities are not regularly synced back to the vgic_irq + * state, they are explicitly synced back here. This is to ensure that + * any pending PPIs are evaluated correctly when first running the guest + * after setting the state. + */ + for_ppi_state(vcpu, reg, 8, + irq->priority = (u8)(val >> (8 * i)); + ); + + /* + * Update the state that will be written to the ICH_PPI_PRIORITYRx_EL2 + * on next guest entry. + */ + cpu_if->vgic_ppi_priorityr[reg] = val; + + return 0; +} + +static int get_gic_ppi_priorityr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int reg = ((r->CRm & 0x1) << 3) + r->Op2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) { + *val = 0; + return 0; + } + + /* + * The priorities are only synced back to the vgic_irq state when the + * vcpu is entering WFI (KVM only needs to know the priorities when + * evaluating if there are pending PPI interrupts for a vcpu). The raw + * register ICH_PPI_PRIORITYRx_EL1 state is simply saved and restored + * blindly. This state is just returned as it contains the most recent + * priorities written by the guest. + */ + *val = cpu_if->vgic_ppi_priorityr[reg]; + + return 0; +} + +/* + * The following registers are NOT supported: + * + * - ICC_HAPR_EL1 + * The value of this is directly generated by the GICv5 hardware based on + * the ICC_APR_EL1 when the guest is running. + * - ICC_IAFFIDR_EL1 + * The IAFFID for a GICv5 VPE is the same as the VPE ID, which is the index + * into the in-memory VPE Table. This is not configurable, and instead we + * rely on userspace recreating the VPEs in the same order prior to + * restoring guest state. + * - ICC_PPI_CACTIVER<n>_EL1 + * Only raw state writes are supported via the S(et) variant. + * - ICC_PPI_CPENDR<n>_EL1 + * Only raw state writes are supported via the S(et) variant. + */ +static const struct sys_reg_desc gic_v5_icc_reg_descs[] = { + { SYS_DESC(SYS_ICC_ICSR_EL1), + .set_user = set_gic_icsr, .get_user = get_gic_icsr, }, + { SYS_DESC(SYS_ICC_PPI_ENABLER0_EL1), + .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, }, + { SYS_DESC(SYS_ICC_PPI_ENABLER1_EL1), + .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, }, + /* + * Only ICC_SACTIVER<n>_EL1 is exposed to the guest. This is treated as + * a *RAW* write of register state for writes. + */ + { SYS_DESC(SYS_ICC_PPI_SACTIVER0_EL1), + .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, }, + { SYS_DESC(SYS_ICC_PPI_SACTIVER1_EL1), + .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, }, + /* + * Only ICC_SPENDR<n>_EL1 is exposed to the guest. This is treated as + * a *RAW* write of register state for writes. + */ + { SYS_DESC(SYS_ICC_PPI_SPENDR0_EL1), + .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, }, + { SYS_DESC(SYS_ICC_PPI_SPENDR1_EL1), + .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR0_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR1_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR2_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR3_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR4_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR5_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR6_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR7_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR8_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR9_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR10_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR11_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR12_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR13_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR14_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR15_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_APR_EL1), + .set_user = set_gic_apr, .get_user = get_gic_apr, }, + { SYS_DESC(SYS_ICC_CR0_EL1), + .set_user = set_gic_cr0, .get_user = get_gic_cr0, }, + { SYS_DESC(SYS_ICC_PCR_EL1), + .set_user = set_gic_pcr, .get_user = get_gic_pcr, }, +}; + +const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz) +{ + *sz = ARRAY_SIZE(gic_v5_icc_reg_descs); + return gic_v5_icc_reg_descs; +} + +static u64 attr_to_id(u64 attr) +{ + return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP1_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRN_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRM_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP2_MASK, attr)); +} + +int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) +{ + const struct sys_reg_desc *r; + + r = get_reg_by_id(attr_to_id(attr->attr), gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); + + if (r && !sysreg_hidden(vcpu, r)) + return 0; + + return -ENXIO; +} + +int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr, + bool is_write) +{ + struct kvm_one_reg reg = { + .id = attr_to_id(attr->attr), + .addr = attr->addr, + }; + + if (is_write) + return kvm_sys_reg_set_user(vcpu, ®, gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); + else + return kvm_sys_reg_get_user(vcpu, ®, gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); +}
diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index a58575d..979301c 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c
@@ -84,6 +84,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) !kvm_vgic_global_state.can_emulate_gicv2) return -ENODEV; + if (kvm_vm_hyp_is_distrusting(kvm) && type != KVM_DEV_TYPE_ARM_VGIC_V3) + return -ENODEV; /* * Ensure mutual exclusion with vCPU creation and any vCPU ioctls by: * @@ -119,19 +121,34 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } - kvm_for_each_vcpu(i, vcpu, kvm) { - if (vcpu_has_run_once(vcpu)) - goto out_unlock; - } + if (kvm_id_regs_final(kvm)) + goto out_unlock; ret = 0; - if (type == KVM_DEV_TYPE_ARM_VGIC_V2) + switch (type) { + case KVM_DEV_TYPE_ARM_VGIC_V2: kvm->max_vcpus = VGIC_V2_MAX_CPUS; - else if (type == KVM_DEV_TYPE_ARM_VGIC_V3) + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: kvm->max_vcpus = VGIC_V3_MAX_CPUS; - else if (type == KVM_DEV_TYPE_ARM_VGIC_V5) - kvm->max_vcpus = min(VGIC_V5_MAX_CPUS, - kvm_vgic_global_state.max_gic_vcpus); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->max_vcpus = kvm_vgic_global_state.max_gicv5_vcpus; + break; + } + + /* + * KVM_CREATE_VCPU enforces the model-specific limit if the VGIC has + * already been created. Apply the same limit to any existing vCPUs so + * that the result does not depend on the order in which userspace + * creates them. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu->vcpu_id >= kvm->max_vcpus) { + ret = -E2BIG; + goto out_unlock; + } + } if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) { ret = -E2BIG; @@ -150,6 +167,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) case KVM_DEV_TYPE_ARM_VGIC_V3: INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions); break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; } /* @@ -164,29 +183,49 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) break; } - if (ret) { - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - kfree(vgic_cpu->private_irqs); - vgic_cpu->private_irqs = NULL; - } - - kvm->arch.vgic.vgic_model = 0; - kvm->arch.vgic.in_kernel = false; - goto out_unlock; - } + if (ret) + goto out_free_private_irqs; if (type == KVM_DEV_TYPE_ARM_VGIC_V3) kvm->arch.vgic.nassgicap = system_supports_direct_sgis(); - /* - * We now know that we have a GICv5. The Arch Timer PPI interrupts may - * have been initialised at this stage, but will have done so assuming - * that we have an older GIC, meaning that the IntIDs won't be - * correct. We init them again, and this time they will be correct. - */ - if (type == KVM_DEV_TYPE_ARM_VGIC_V5) + if (type == KVM_DEV_TYPE_ARM_VGIC_V5) { + /* Allocate a vIRS for GICv5 systems */ + kvm->arch.vgic.vgic_v5_irs_data = kzalloc_obj(struct vgic_v5_irs, + GFP_KERNEL_ACCOUNT); + if (!kvm->arch.vgic.vgic_v5_irs_data) { + ret = -ENOMEM; + goto out_free_private_irqs; + } + + /* + * Initialization happens later, for now just explicitly + * disable the device and undef its base address. + */ + kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base = VGIC_ADDR_UNDEF; + + /* + * We now know that we have a GICv5. The Arch Timer PPI + * interrupts may have been initialised at this stage, but will + * have done so assuming that we have an older GIC, meaning that + * the IntIDs won't be correct. We init them again, and this + * time they will be correct. + */ kvm_timer_init_vm(kvm); + } + + goto out_unlock; + +out_free_private_irqs: + kvm_for_each_vcpu(i, vcpu, kvm) { + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + + kfree(vgic_cpu->private_irqs); + vgic_cpu->private_irqs = NULL; + } + + kvm->arch.vgic.in_kernel = false; + kvm->arch.vgic.vgic_model = 0; out_unlock: mutex_unlock(&kvm->arch.config_lock); @@ -393,15 +432,27 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) if (ret) return ret; - /* - * If we are creating a VCPU with a GICv3 we must also register the - * KVM io device for the redistributor that belongs to this VCPU. - */ - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V3: + /* + * If we are creating a VCPU with a GICv3 we must also register + * the KVM io device for the redistributor that belongs to this + * VCPU. + */ mutex_lock(&vcpu->kvm->slots_lock); ret = vgic_register_redist_iodev(vcpu); mutex_unlock(&vcpu->kvm->slots_lock); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + /* + * Ensure that it is possible to represent the + * userspace-allocated vcpu_id in the hardware-limited (or + * KVM-capped) VPE table used by GICv5. + */ + if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus) + return -EINVAL; } + return ret; } @@ -461,6 +512,9 @@ int vgic_init(struct kvm *kvm) return ret; } } else { + if (!dist->nr_spis) + dist->nr_spis = VGIC_V5_DEFAULT_NR_SPIS; + ret = vgic_v5_init(kvm); if (ret) return ret; @@ -470,8 +524,12 @@ int vgic_init(struct kvm *kvm) kvm_vgic_vcpu_reset(vcpu); ret = kvm_vgic_setup_default_irq_routing(kvm); - if (ret) + if (ret) { + if (vgic_is_v5(kvm)) + vgic_v5_teardown(kvm); + return ret; + } vgic_debug_init(kvm); dist->initialized = true; @@ -492,16 +550,24 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) dist->nr_spis = 0; dist->vgic_dist_base = VGIC_ADDR_UNDEF; - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V2: + dist->vgic_cpu_base = VGIC_ADDR_UNDEF; + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) vgic_v3_free_redist_region(kvm, rdreg); INIT_LIST_HEAD(&dist->rd_regions); - } else { - dist->vgic_cpu_base = VGIC_ADDR_UNDEF; - } - if (vgic_supports_direct_irqs(kvm)) - vgic_v4_teardown(kvm); + if (vgic_supports_direct_irqs(kvm)) + vgic_v4_teardown(kvm); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + vgic_v5_teardown(kvm); + kfree(dist->vgic_v5_irs_data); + dist->vgic_v5_irs_data = NULL; + break; + } xa_destroy(&dist->lpi_xa); } @@ -620,9 +686,8 @@ int vgic_lazy_init(struct kvm *kvm) int kvm_vgic_map_resources(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - bool needs_dist = true; enum vgic_type type; - gpa_t dist_base; + gpa_t dist_base, irs_base; int ret = 0; if (likely(smp_load_acquire(&dist->ready))) @@ -645,13 +710,12 @@ int kvm_vgic_map_resources(struct kvm *kvm) } else { ret = vgic_v5_map_resources(kvm); type = VGIC_V5; - needs_dist = false; } if (ret) goto out; - if (needs_dist) { + if (type != VGIC_V5) { dist_base = dist->vgic_dist_base; mutex_unlock(&kvm->arch.config_lock); @@ -661,7 +725,19 @@ int kvm_vgic_map_resources(struct kvm *kvm) goto out_slots; } } else { + irs_base = dist->vgic_v5_irs_data->vgic_v5_irs_base; mutex_unlock(&kvm->arch.config_lock); + + if (IS_VGIC_ADDR_UNDEF(irs_base)) { + ret = -ENXIO; + goto out_slots; + } + + ret = vgic_v5_register_irs_iodev(kvm, irs_base); + if (ret) { + kvm_err("Unable to register VGIC IRS MMIO regions\n"); + goto out_slots; + } } smp_store_release(&dist->ready, true);
diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c index 19a1094..ee1a1b2 100644 --- a/arch/arm64/kvm/vgic/vgic-irqfd.c +++ b/arch/arm64/kvm/vgic/vgic-irqfd.c
@@ -19,9 +19,14 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { - unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + unsigned int spi_id; int ret; + if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) + spi_id = vgic_v5_make_spi(e->irqchip.pin); + else + spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + if (!vgic_valid_spi(kvm, spi_id)) return -EINVAL; @@ -45,6 +50,13 @@ int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { + /* + * The common routing table reserves enough entries for all GICv5 SPI + * pins. KVM limits GICv5 to 1024 SPI pins, while GICv2 and GICv3 + * only support SPIs 32..1019, leaving 988 pins. + */ + unsigned int max_irqchip_pins = vgic_is_v5(kvm) ? + VGIC_V5_MAX_NR_SPIS : VGIC_MAX_SPI - VGIC_NR_PRIVATE_IRQS + 1; int r = -EINVAL; switch (ue->type) { @@ -52,8 +64,8 @@ int kvm_set_routing_entry(struct kvm *kvm, e->set = vgic_irqfd_set_irq; e->irqchip.irqchip = ue->u.irqchip.irqchip; e->irqchip.pin = ue->u.irqchip.pin; - if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) || - (e->irqchip.irqchip >= KVM_NR_IRQCHIPS)) + if (e->irqchip.pin >= max_irqchip_pins || + e->irqchip.irqchip >= KVM_NR_IRQCHIPS) goto out; break; case KVM_IRQ_ROUTING_MSI:
diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c new file mode 100644 index 0000000..1874c3c --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c
@@ -0,0 +1,1217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 ARM Limited, All Rights Reserved. + */ +#include <linux/bitops.h> +#include <linux/bsearch.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/kvm.h> +#include <linux/kvm_host.h> +#include <kvm/iodev.h> +#include <kvm/arm_arch_timer.h> +#include <kvm/arm_vgic.h> + +#include "vgic.h" +#include "vgic-mmio.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_dist *vgic_v5_get_vgic(struct kvm_vcpu *vcpu) +{ + return &vcpu->kvm->arch.vgic; +} + +static struct vgic_v5_irs *vgic_v5_get_irs(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm->arch.vgic.vgic_v5_irs_data; +} + +static unsigned long vgic_v5_mmio_read_irs_misc(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct kvm_vcpu *target_vcpu; + u8 vpe_id_bits; + u64 value = 0; + + switch (offset) { + case GICV5_IRS_IDR0: + value = FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, irs->idr0.domain); + value |= FIELD_PREP(GICV5_IRS_IDR0_PA_RANGE, irs->idr0.pa_range); + if (irs->idr0.virt) + value |= GICV5_IRS_IDR0_VIRT; + if (irs->idr0.setlpi) + value |= GICV5_IRS_IDR0_SETLPI; + if (irs->idr0.mec) + value |= GICV5_IRS_IDR0_MEC; + if (irs->idr0.mpam) + value |= GICV5_IRS_IDR0_MPAM; + if (irs->idr0.swe) + value |= GICV5_IRS_IDR0_SWE; + value |= FIELD_PREP(GICV5_IRS_IDR0_IRSID, irs->idr0.irs_id); + break; + case GICV5_IRS_IDR1: + value = FIELD_PREP(GICV5_IRS_IDR1_PE_CNT, + atomic_read(&vcpu->kvm->online_vcpus)); + /* + * IRS_IDR1 encodes IAFFID_BITS as N - 1. + */ + vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu); + value |= FIELD_PREP(GICV5_IRS_IDR1_IAFFID_BITS, vpe_id_bits - 1); + value |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, irs->idr1.priority_bits); + break; + case GICV5_IRS_IDR2: + value = FIELD_PREP(GICV5_IRS_IDR2_ISTMD_SZ, irs->idr2.istmd_sz); + if (irs->idr2.istmd) + value |= GICV5_IRS_IDR2_ISTMD; + value |= FIELD_PREP(GICV5_IRS_IDR2_IST_L2SZ, irs->idr2.ist_l2sz); + if (irs->idr2.ist_levels) + value |= GICV5_IRS_IDR2_IST_LEVELS; + value |= FIELD_PREP(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, irs->idr2.min_lpi_id_bits); + value |= GICV5_IRS_IDR2_LPI; + value |= FIELD_PREP(GICV5_IRS_IDR2_ID_BITS, irs->idr2.id_bits); + break; + case GICV5_IRS_IDR5: + value = FIELD_PREP(GICV5_IRS_IDR5_SPI_RANGE, irs->idr5.spi_range); + break; + case GICV5_IRS_IDR6: + value = FIELD_PREP(GICV5_IRS_IDR6_SPI_IRS_RANGE, irs->idr6.spi_irs_range); + break; + case GICV5_IRS_IDR7: + value = FIELD_PREP(GICV5_IRS_IDR7_SPI_BASE, irs->idr7.spi_base); + break; + case GICV5_IRS_IIDR: + /* Revision, Variant, ProductID are implementation defined */ + value = FIELD_PREP(GICV5_IRS_IIDR_PRODUCT_ID, PRODUCT_ID_KVM); + value |= FIELD_PREP(GICV5_IRS_IIDR_VARIANT, 0); + value |= FIELD_PREP(GICV5_IRS_IIDR_REVISION, 0); + value |= FIELD_PREP(GICV5_IRS_IIDR_IMPLEMENTER, IMPLEMENTER_ARM); + break; + case GICV5_IRS_AIDR: + value = FIELD_PREP(GICV5_IRS_AIDR_COMPONENT, + GICV5_AIDR_COMPONENT_IRS); + value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMAJORREV, + GICV5_AIDR_ARCH_MAJ_REV_V5); + value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMINORREV, + GICV5_AIDR_ARCH_MIN_REV_V0); + break; + case GICV5_IRS_CR0: + /* + * The IRS is ALWAYS idle as we handle things instantaneously + * from a guest's viewpoint. + */ + value = GICV5_IRS_CR0_IDLE; + if (READ_ONCE(vcpu->kvm->arch.vgic.enabled)) + value |= GICV5_IRS_CR0_IRSEN; + break; + case GICV5_IRS_CR1: + if (irs->cr1.vped_wa) + value |= GICV5_IRS_CR1_VPED_WA; + if (irs->cr1.vped_ra) + value |= GICV5_IRS_CR1_VPED_RA; + if (irs->cr1.vmd_wa) + value |= GICV5_IRS_CR1_VMD_WA; + if (irs->cr1.vmd_ra) + value |= GICV5_IRS_CR1_VMD_RA; + if (irs->cr1.vpet_ra) + value |= GICV5_IRS_CR1_VPET_RA; + if (irs->cr1.vmt_ra) + value |= GICV5_IRS_CR1_VMT_RA; + if (irs->cr1.ist_wa) + value |= GICV5_IRS_CR1_IST_WA; + if (irs->cr1.ist_ra) + value |= GICV5_IRS_CR1_IST_RA; + value |= FIELD_PREP(GICV5_IRS_CR1_IC, irs->cr1.ic); + value |= FIELD_PREP(GICV5_IRS_CR1_OC, irs->cr1.oc); + value |= FIELD_PREP(GICV5_IRS_CR1_SH, irs->cr1.sh); + break; + case GICV5_IRS_SYNC_STATUSR: + value = GICV5_IRS_SYNC_STATUSR_IDLE; + break; + case GICV5_IRS_PE_SELR: + value = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, + READ_ONCE(irs->pe_selr.iaffid)); + break; + case GICV5_IRS_PE_STATUSR: + /* We assume that the PE is Online if present. Always IDLE too */ + value = GICV5_IRS_PE_STATUSR_IDLE; + + /* Set ONLINE and V if IAFFID selects a present PE */ + if (kvm_get_vcpu_by_id(vcpu->kvm, + READ_ONCE(irs->pe_selr.iaffid))) { + value |= GICV5_IRS_PE_STATUSR_ONLINE; + value |= GICV5_IRS_PE_STATUSR_V; + } + break; + case GICV5_IRS_PE_CR0: { + u32 iaffid = READ_ONCE(irs->pe_selr.iaffid); + + /* + * Make sure that we are doing something reasonable first. + * Remember, the IAFFID is the same as the VPE_ID + */ + target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, iaffid); + if (!target_vcpu) { + kvm_debug("Guest programmed invalid IAFFID (0x%x) into the IRS_PE_SELR\n", + iaffid); + break; + } + + value = GICV5_IRS_PE_CR0_DPS; + break; + } + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_misc(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + struct vgic_dist *vgic = vgic_v5_get_vgic(vcpu); + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_CR0: + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + bool was_enabled = vgic->enabled; + + WRITE_ONCE(vgic->enabled, !!(val & GICV5_IRS_CR0_IRSEN)); + + /* Reload other vCPUs on an edge */ + if (was_enabled != !!(val & GICV5_IRS_CR0_IRSEN)) + kvm_make_all_cpus_request(vcpu->kvm, + KVM_REQ_RELOAD_GICv5); + } + return; + case GICV5_IRS_CR1: + irs->cr1.sh = FIELD_GET(GICV5_IRS_CR1_SH, val); + irs->cr1.oc = FIELD_GET(GICV5_IRS_CR1_OC, val); + irs->cr1.ic = FIELD_GET(GICV5_IRS_CR1_IC, val); + irs->cr1.ist_ra = !!(val & GICV5_IRS_CR1_IST_RA); + irs->cr1.ist_wa = !!(val & GICV5_IRS_CR1_IST_WA); + irs->cr1.vmt_ra = !!(val & GICV5_IRS_CR1_VMT_RA); + irs->cr1.vpet_ra = !!(val & GICV5_IRS_CR1_VPET_RA); + irs->cr1.vmd_ra = !!(val & GICV5_IRS_CR1_VMD_RA); + irs->cr1.vmd_wa = !!(val & GICV5_IRS_CR1_VMD_WA); + irs->cr1.vped_ra = !!(val & GICV5_IRS_CR1_VPED_RA); + irs->cr1.vped_wa = !!(val & GICV5_IRS_CR1_VPED_WA); + return; + case GICV5_IRS_PE_SELR: + WRITE_ONCE(irs->pe_selr.iaffid, + FIELD_GET(GICV5_IRS_PE_SELR_IAFFID, val)); + return; + case GICV5_IRS_PE_CR0: + /* + * We actually have nothing to do here as we don't support + * 1-of-N routing. The only thing that the guest can correctly + * write here is 0x1. However, there's no way to fault if it + * writes something else. This is effectively a WI in our case, + * but we keep it here for the purposes of documenting it. + */ + return; + default: + return; + } +} + +static bool vgic_v5_is_spi_selr_valid(struct vgic_v5_irs *irs, u32 id) +{ + /* Invalid - we don't have any SPIs at all */ + if (irs->idr5.spi_range == 0) + return false; + + /* Invalid - we don't have any on this IRS */ + if (irs->idr6.spi_irs_range == 0) + return false; + + /* Invalid - ID is less than min */ + if (id < irs->idr7.spi_base) + return false; + + /* Invalid - ID is greater than max */ + if (id >= (irs->idr7.spi_base + irs->idr6.spi_irs_range)) + return false; + + return true; +} + +static unsigned long vgic_v5_mmio_read_irs_spi(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct vgic_irq *irq; + u64 value = 0; + + switch (offset) { + case GICV5_IRS_SPI_SELR: + /* Return whatever was last written */ + value = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, READ_ONCE(irs->spi_selr.id)); + break; + case GICV5_IRS_SPI_STATUSR: + /* We assume that we can always claim to be idle */ + value = GICV5_IRS_SPI_STATUSR_IDLE; + if (vgic_v5_is_spi_selr_valid(irs, READ_ONCE(irs->spi_selr.id))) + value |= GICV5_IRS_SPI_STATUSR_V; + break; + case GICV5_IRS_SPI_DOMAINR: + value = FIELD_PREP(GICV5_IRS_SPI_DOMAINR_DOMAIN, + GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE); + break; + case GICV5_IRS_SPI_CFGR: { + u32 id = READ_ONCE(irs->spi_selr.id); + + if (!vgic_v5_is_spi_selr_valid(irs, id)) { + /* Fault with IRS_SPI_SELR; return 0*/ + value = 0; + break; + } + + irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id)); + if (!irq) { + kvm_debug_ratelimited("Guest trying to access SPI not backed by KVM\n"); + value = 0; + break; + } + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (irq->config == VGIC_CONFIG_LEVEL) + value = GICV5_IRS_SPI_CFGR_TM; + } + + vgic_put_irq(vcpu->kvm, irq); + + break; + } + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_spi(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct vgic_irq *irq; + + switch (offset) { + case GICV5_IRS_SPI_SELR: + WRITE_ONCE(irs->spi_selr.id, + FIELD_GET(GICV5_IRS_SPI_SELR_ID, val)); + return; + case GICV5_IRS_SPI_CFGR: { + u32 id = READ_ONCE(irs->spi_selr.id); + + if (!vgic_v5_is_spi_selr_valid(irs, id)) + return; + + /* + * Find KVM's representation of the interrupt - we need to make + * sure that KVM's view agrees with the guest's, else interrupt + * injection won't work properly for level-triggered interrupts + * (we fail to handle the clearing of the pending state if KVM + * thinks that the interrupt is edge-triggered, which is the + * default.) + */ + irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id)); + if (!irq) + return; + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (val & GICV5_IRS_SPI_CFGR_TM) + irq->config = VGIC_CONFIG_LEVEL; + else + irq->config = VGIC_CONFIG_EDGE; + } + + vgic_put_irq(vcpu->kvm, irq); + + return; + } + default: + return; + } +} + +static bool vgic_v5_ist_cfgr_valid(struct vgic_v5_irs *irs) +{ + unsigned int expected_istsz; + + if (irs->ist_cfgr.lpi_id_bits < irs->idr2.min_lpi_id_bits || + irs->ist_cfgr.lpi_id_bits > irs->idr2.id_bits) + return false; + + if (!irs->idr2.istmd) + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_4; + else if (irs->ist_cfgr.lpi_id_bits >= irs->idr2.istmd_sz) + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_16; + else + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_8; + + if (irs->ist_cfgr.istsz != expected_istsz) + return false; + + if (irs->ist_cfgr.structure && !irs->idr2.ist_levels) + return false; + + if (!irs->ist_cfgr.structure) + return true; + + return irs->ist_cfgr.l2sz == irs->idr2.ist_l2sz; +} + +static u64 vgic_v5_read_irs_ist_baser(struct vgic_v5_irs *irs) +{ + u64 value; + + value = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + irs->ist_baser.addr >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + if (irs->ist_baser.valid) + value |= GICV5_IRS_IST_BASER_VALID; + + return value; +} + +static bool vgic_v5_ist_cfgr_matches(const struct vgic_v5_irs *irs, + unsigned long val) +{ + u8 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + u8 l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + u8 istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + bool structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + + return irs->ist_cfgr.lpi_id_bits == lpi_id_bits && + irs->ist_cfgr.l2sz == l2sz && + irs->ist_cfgr.istsz == istsz && + irs->ist_cfgr.structure == structure; +} + +static unsigned long vgic_v5_mmio_read_irs_ist(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + u64 value = 0; + + switch (offset) { + case GICV5_IRS_IST_STATUSR: + return GICV5_IRS_IST_STATUSR_IDLE; + case GICV5_IRS_IST_CFGR: + if (irs->ist_cfgr.structure) + value |= GICV5_IRS_IST_CFGR_STRUCTURE; + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, irs->ist_cfgr.istsz); + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, irs->ist_cfgr.l2sz); + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, irs->ist_cfgr.lpi_id_bits); + break; + case GICV5_IRS_IST_BASER: + case GICV5_IRS_IST_BASER + 4: + value = vgic_v5_read_irs_ist_baser(irs); + return extract_bytes(value, offset & 7, len); + default: + return 0; + } + + return value; +} + +static bool vgic_v5_ist_baser_matches(const struct vgic_v5_irs *irs, + unsigned long val) +{ + u64 addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; + bool valid = !!(val & GICV5_IRS_IST_BASER_VALID); + + return irs->ist_baser.addr == addr && irs->ist_baser.valid == valid; +} + +static void vgic_v5_update_irs_ist_baser(struct vgic_v5_irs *irs, + unsigned long val) +{ + irs->ist_baser.valid = !!(val & GICV5_IRS_IST_BASER_VALID); + irs->ist_baser.addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; +} + +static int vgic_v5_write_irs_ist_baser(struct kvm_vcpu *vcpu, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + enum gicv5_vcpu_cmd cmd = LPI_VIST_MAKE_INVALID; + bool valid = !!(val & GICV5_IRS_IST_BASER_VALID); + int rc; + + /* The address cannot be changed while the IST is valid. */ + if (irs->ist_baser.valid && valid) + return 0; + + /* Valid -> Invalid */ + if (irs->ist_baser.valid && !valid) { + /* Make the LPI IST invalid and then ... */ + rc = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu), &cmd); + if (rc) + return rc; + + vgic_v5_update_irs_ist_baser(irs, val); + + /* + * ... reflect that in the emulated BASER before freeing the + * host IST. If the free fails, the guest-visible valid bit + * still matches the hardware state. + */ + return vgic_v5_lpi_ist_free(vcpu->kvm); + } else if (!irs->ist_baser.valid && valid) { /* Invalid -> Valid */ + if (!vgic_v5_ist_cfgr_valid(irs)) + return -EINVAL; + + rc = vgic_v5_lpi_ist_alloc(vcpu->kvm, irs->ist_cfgr.lpi_id_bits); + if (rc) + return rc; + } + + /* Now that we've handled the edges, update the valid bit and addr */ + vgic_v5_update_irs_ist_baser(irs, val); + + return 0; +} + +static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_IST_CFGR: + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + if (irs->ist_baser.valid) + return; + + irs->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + irs->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + irs->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + irs->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + } + return; + case GICV5_IRS_IST_BASER: + case GICV5_IRS_IST_BASER + 4: { + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + u64 baser; + + baser = update_64bit_reg(vgic_v5_read_irs_ist_baser(irs), + offset & 4, len, val); + vgic_v5_write_irs_ist_baser(vcpu, baser); + } + return; + } + default: + return; + } +} + +static unsigned long vgic_v5_coresight_read(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_CORESIGHT_DEVARCH: + return GICV5_CORESIGHT_DEVARCH_VAL; + case GICV5_CORESIGHT_PIDR4: + return GICV5_CORESIGHT_PIDR4_JEP106_CONT; + case GICV5_CORESIGHT_PIDR5: + return GICV5_CORESIGHT_PIDR5_RES0; + case GICV5_CORESIGHT_PIDR6: + return GICV5_CORESIGHT_PIDR6_RES0; + case GICV5_CORESIGHT_PIDR7: + return GICV5_CORESIGHT_PIDR7_RES0; + case GICV5_CORESIGHT_PIDR0: + return GICV5_CORESIGHT_PIDR0_PART_0; + case GICV5_CORESIGHT_PIDR1: + return GICV5_CORESIGHT_PIDR1_DES_0_PART_1; + case GICV5_CORESIGHT_PIDR2: + return GICV5_CORESIGHT_PIDR2_DES_1; + case GICV5_CORESIGHT_PIDR3: + return GICV5_CORESIGHT_PIDR3_REVAND_CMOD; + case GICV5_CORESIGHT_CIDR0: + return GICV5_CORESIGHT_CIDR0_VAL; + case GICV5_CORESIGHT_CIDR1: + return GICV5_CORESIGHT_CIDR1_VAL; + case GICV5_CORESIGHT_CIDR2: + return GICV5_CORESIGHT_CIDR2_VAL; + case GICV5_CORESIGHT_CIDR3: + return GICV5_CORESIGHT_CIDR3_VAL; + default: + return 0; + } +} + +static unsigned long vgic_v5_mmio_uaccess_read_irs_status(struct kvm_vcpu *vcpu, + gpa_t addr, + unsigned int len) +{ + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_SYNC_STATUSR: + return GICV5_IRS_SYNC_STATUSR_IDLE; + case GICV5_IRS_SPI_STATUSR: + return GICV5_IRS_SPI_STATUSR_IDLE; + case GICV5_IRS_PE_STATUSR: + return GICV5_IRS_PE_STATUSR_IDLE; + case GICV5_IRS_IST_STATUSR: + return GICV5_IRS_IST_STATUSR_IDLE; + default: + return 0; + } +} + +static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_dist *vgic = &vcpu->kvm->arch.vgic; + struct vgic_v5_irs *irs_data = vgic->vgic_v5_irs_data; + size_t offset = addr & (SZ_64K - 1); + int ret; + + /* + * The following registers are ONLY settable via uaccesses. The guest + * cannot write them! + */ + + switch (offset) { + case GICV5_IRS_IDR0: + if (FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val) != + GICV5_IRS_IDR0_INT_DOM_NON_SECURE) + return -EINVAL; + + if ((val & GICV5_IRS_IDR0_VIRT) || + (val & GICV5_IRS_IDR0_ONE_N) || + (val & GICV5_IRS_IDR0_VIRT_ONE_N) || + (val & GICV5_IRS_IDR0_SETLPI) || + (val & GICV5_IRS_IDR0_MEC) || + (val & GICV5_IRS_IDR0_MPAM) || + (val & GICV5_IRS_IDR0_SWE)) + return -EINVAL; + + irs_data->idr0.domain = FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val); + irs_data->idr0.pa_range = FIELD_GET(GICV5_IRS_IDR0_PA_RANGE, val); + irs_data->idr0.virt = !!(val & GICV5_IRS_IDR0_VIRT); + irs_data->idr0.setlpi = !!(val & GICV5_IRS_IDR0_SETLPI); + irs_data->idr0.mec = !!(val & GICV5_IRS_IDR0_MEC); + irs_data->idr0.mpam = !!(val & GICV5_IRS_IDR0_MPAM); + irs_data->idr0.swe = !!(val & GICV5_IRS_IDR0_SWE); + irs_data->idr0.irs_id = FIELD_GET(GICV5_IRS_IDR0_IRSID, val); + break; + case GICV5_IRS_IDR1: { + unsigned int iaffid_bits, priority_bits; + u8 vpe_id_bits; + + /* Ignore writes to PE_CNT as this is populated from num vcpus */ + iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, val); + priority_bits = FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, val); + + /* IAFFID_BITS is derived from the VMTE and encoded as N - 1. */ + vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu); + if (iaffid_bits != vpe_id_bits - 1) + return -EINVAL; + + if (priority_bits > gicv5_global_data.irs_pri_bits - 1) + return -EINVAL; + + irs_data->idr1.priority_bits = priority_bits; + break; + } + case GICV5_IRS_IDR2: + /* We always support LPIs */ + if (!(val & GICV5_IRS_IDR2_LPI)) + return -EINVAL; + + /* We only support LPIs with linear, non-metadata guest ISTs */ + if (val & GICV5_IRS_IDR2_IST_LEVELS) + return -EINVAL; + + if ((val & GICV5_IRS_IDR2_ISTMD) || + FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val)) + return -EINVAL; + + /* We can't present more bits than we have support for in HW */ + if (FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val) > + vgic_v5_irs_ist_id_bits(&irs_caps)) + return -EINVAL; + + /* Min LPI ID bits must be greater than or equal to the HW */ + if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) < + vgic_v5_irs_min_lpi_id_bits(&irs_caps)) + return -EINVAL; + + if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) > + FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val)) + return -EINVAL; + + irs_data->idr2.istmd_sz = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val); + irs_data->idr2.istmd = !!(val & GICV5_IRS_IDR2_ISTMD); + irs_data->idr2.ist_l2sz = FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, val); + irs_data->idr2.ist_levels = !!(val & GICV5_IRS_IDR2_IST_LEVELS); + irs_data->idr2.min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val); + irs_data->idr2.id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val); + break; + case GICV5_IRS_IDR5: + if (FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, val) != irs_data->idr5.spi_range) + return -EINVAL; + break; + case GICV5_IRS_IDR6: + if (FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, val) != irs_data->idr6.spi_irs_range) + return -EINVAL; + break; + case GICV5_IRS_IDR7: + if (FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, val) != irs_data->idr7.spi_base) + return -EINVAL; + break; + case GICV5_IRS_IST_BASER: + ret = vgic_v5_lpi_ist_exists(vcpu->kvm); + if (ret < 0) + return ret; + + if (ret && !vgic_v5_ist_baser_matches(irs_data, val)) + return -EINVAL; + + if (!irs_data->ist_baser.valid && + (val & GICV5_IRS_IST_BASER_VALID)) + irs_data->lpi_ist_restore_pending = true; + + vgic_v5_update_irs_ist_baser(irs_data, val); + break; + case GICV5_IRS_IST_CFGR: + ret = vgic_v5_lpi_ist_exists(vcpu->kvm); + if (ret < 0) + return ret; + + if (ret && !vgic_v5_ist_cfgr_matches(irs_data, val)) + return -EINVAL; + + irs_data->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + irs_data->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + irs_data->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + irs_data->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + break; + case GICV5_IRS_CR0: + vgic->enabled = !!(val & GICV5_IRS_CR0_IRSEN); + break; + case GICV5_IRS_SPI_CFGR: + break; + case GICV5_IRS_IIDR: + fallthrough; + case GICV5_IRS_AIDR: + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct vgic_register_region vgic_v5_irs_registers[] = { + /* + * This is the IRS_CONFIG_FRAME. + */ + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR0, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR1, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR2, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR3, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR4, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR5, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR6, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR7, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_AIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_CR0, + vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR1, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNCR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SYNC_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_SELR, vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_DOMAINR, vgic_v5_mmio_read_irs_spi, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_RESAMPLER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_CFGR, + vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_STATUSR, + vgic_v5_mmio_read_irs_spi, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_SELR, + vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_PE_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_CR0, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_BASER, + vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, NULL, + vgic_v5_mmio_uaccess_write_irs, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_CFGR, + vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_STATUSR, + vgic_v5_mmio_read_irs_ist, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MAP_L2_ISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + + /* + * The following registers are only for running VMs. They are not yet + * supported as we don't currently support nested, so expose them as + * read-as-zero/write-ignored. + */ + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_BASER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_CFGR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_SELR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_DBR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_HPPIR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_CR0, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_DBR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_SELR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VMTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VPER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VM_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + + /* MEC, MPAM, SWERR - all unimplemented */ + + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_IDR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_MECID_R, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_IDR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_PARTID_R, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER0, + vgic_mmio_read_raz, vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER1, + vgic_mmio_read_raz, vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + + /* CoreSight identification registers */ + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_DEVARCH, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR4, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR5, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR6, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR7, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR0, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR1, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR2, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR3, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR0, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR1, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR2, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR3, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), +}; + +unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev) +{ + dev->regions = vgic_v5_irs_registers; + dev->nr_regions = ARRAY_SIZE(vgic_v5_irs_registers); + + kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); + + /* We represent both of the IRS frames back to back, so this is 128K */ + return KVM_VGIC_V5_IRS_SIZE; +} + +int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address) +{ + struct vgic_io_device *io_device = &kvm->arch.vgic.vgic_v5_irs_data->iodev; + unsigned int len; + + /* + * Design choice: Force MMIO region to be 64k aligned. Simplifies + * pulling out registers. + */ + if (!IS_ALIGNED(irs_base_address, SZ_64K)) + return -EINVAL; + + len = vgic_v5_init_irs_iodev(io_device); + + io_device->base_addr = irs_base_address; + io_device->iodev_type = IODEV_GICV5_IRS; + io_device->redist_vcpu = NULL; + + return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, irs_base_address, len, + &io_device->dev); +} + +/** + * kvm_vgic_v5_irs_init: initialize the IRS data structures + * @kvm: kvm struct pointer + * @nr_spis: number of spis, frozen by caller + */ +int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct vgic_v5_irs *irs = dist->vgic_v5_irs_data; + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + size_t nr_spi_bits; + u64 mmfr0; + int ret, i; + + /* + * We (KVM) allocate an Interrupt State Table (IST) for SPIs. The + * hardware mandates that lower 6 bits of the address are 0. Each ISTE + * is 4 bytes in size (or larger if metadata storage is required), so 16 + * entries would be enough for alignment. We require userspace to + * provide at least 32 SPIs, so we always meet alignment requirements. + */ + if (nr_spis) { + dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), + GFP_KERNEL_ACCOUNT); + if (!dist->spis) + return -ENOMEM; + + /* + * In the following code we do not take the irq struct lock since + * no other action on irq structs can happen while the VGIC is + * not initialized yet. + */ + for (i = 0; i < nr_spis; i++) { + struct vgic_irq *irq = &dist->spis[i]; + + irq->intid = vgic_v5_make_spi(i); + INIT_LIST_HEAD(&irq->ap_list); + raw_spin_lock_init(&irq->irq_lock); + irq->vcpu = NULL; + irq->target_vcpu = vcpu0; + refcount_set(&irq->refcount, 0); + /* + * The guest controls the enable state, and again it is + * directly handled by the hardware. From our point of + * view it is always enabled. + */ + irq->enabled = 1; + vgic_v5_set_spi_ops(irq); + } + + nr_spi_bits = fls(roundup_pow_of_two(nr_spis)) - 1; + + ret = vgic_v5_spi_ist_alloc(kvm, nr_spi_bits); + if (ret) { + kfree(dist->spis); + dist->spis = NULL; + return ret; + } + } + + /* Set sane initial state for the IRS MMIO registers */ + + irs->idr0.domain = GICV5_IRS_IDR0_INT_DOM_NON_SECURE; + + mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); + irs->idr0.pa_range = cpuid_feature_extract_unsigned_field(mmfr0, + ID_AA64MMFR0_EL1_PARANGE_SHIFT); + + irs->idr0.virt = 0; + irs->idr0.setlpi = 0; + irs->idr0.mec = 0; + irs->idr0.mpam = 0; + irs->idr0.swe = 0; + irs->idr0.irs_id = 0; + + irs->idr1.priority_bits = gicv5_global_data.irs_pri_bits - 1; + + /* + * Support 16-bits of ID space for the IRS. This should be sufficient + * for most applications, and the CPUIF is guaranteed to have at least + * 16-bits of ID space support (we actually present 16-bits there, even + * if the hardware supports more). Warn if the hardware doesn't support + * 16 bits, and use the smaller value. YMMV! + * + * As for the minimum number of ID bits, we match the hardware's + * capability. + */ + if (vgic_v5_irs_ist_id_bits(&irs_caps) < 16) + pr_warn_ratelimited("Host IRS supports fewer than 16 ID bits for ISTs (%u)\n", + vgic_v5_irs_ist_id_bits(&irs_caps)); + + irs->idr2.id_bits = min(16, vgic_v5_irs_ist_id_bits(&irs_caps)); + irs->idr2.min_lpi_id_bits = vgic_v5_irs_min_lpi_id_bits(&irs_caps); + + /* Only allow the guest to create Linear ISTs - simplifies Save/Restore */ + irs->idr2.ist_levels = 0; + irs->idr2.ist_l2sz = GICV5_IRS_IST_CFGR_L2SZ_4K; + irs->idr2.istmd = 0; + irs->idr2.istmd_sz = 0; + + /* We have a single IRS, only. All SPIs reside here! */ + irs->idr5.spi_range = nr_spis; + irs->idr6.spi_irs_range = nr_spis; + irs->idr7.spi_base = 0; + + irs->cr1.sh = 0; + irs->cr1.oc = 0; + irs->cr1.ic = 0; + irs->cr1.ist_ra = 0; + irs->cr1.ist_wa = 0; + irs->cr1.vmt_ra = 0; + irs->cr1.vpet_ra = 0; + irs->cr1.vmd_ra = 0; + irs->cr1.vmd_wa = 0; + irs->cr1.vped_ra = 0; + irs->cr1.vped_wa = 0; + + irs->spi_selr.id = -1; + + irs->pe_selr.iaffid = -1; + + irs->ist_cfgr.lpi_id_bits = 0; + irs->ist_cfgr.l2sz = 0; + irs->ist_cfgr.istsz = 0; + irs->ist_cfgr.structure = 0; + + irs->ist_baser.valid = 0; + irs->ist_baser.addr = 0; + irs->lpi_ist_restore_pending = false; + + return 0; +} + +int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits) +{ + struct vgic_v5_irs *irs = kvm->arch.vgic.vgic_v5_irs_data; + + if (!irs) + return -ENXIO; + + if (!irs->ist_baser.valid) + return 0; + + if (!vgic_v5_ist_cfgr_valid(irs)) + return -EINVAL; + + *id_bits = irs->ist_cfgr.lpi_id_bits; + + return 1; +} + +int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + const struct vgic_register_region *region; + struct vgic_reg_attr reg_attr; + struct kvm_vcpu *vcpu; + gpa_t addr, offset; + int ret, align; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + addr = reg_attr.addr; + + if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS) + return vgic_v5_has_cpu_sysregs_attr(vcpu, attr); + + offset = attr->attr; + + region = vgic_find_mmio_region(vgic_v5_irs_registers, + ARRAY_SIZE(vgic_v5_irs_registers), + offset); + if (!region) + return -ENXIO; + + align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3; + if (offset & align) + return -EINVAL; + + return 0; +} + +/* + * Access the IRS MMIO Regs. Relevant locks have been taken by the calling code. + */ +int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + u64 *reg, bool is_write) +{ + const struct vgic_register_region *region; + gpa_t addr, offset; + unsigned int len; + int align, ret = 0; + + offset = attr->attr; + + if (IS_VGIC_ADDR_UNDEF(dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base)) + return -ENXIO; + + region = vgic_find_mmio_region(vgic_v5_irs_registers, + ARRAY_SIZE(vgic_v5_irs_registers), + offset); + if (!region) + return -ENXIO; + + /* + * Although the spec supports upper/lower 32-bit accesses to + * 64-bit IRS registers, the userspace ABI requires 64-bit + * accesses to all 64-bit wide registers. We therefore only + * support 32-bit accesses to 32-bit-wide registers. + */ + align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3; + len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4; + + if (offset & align) + return -EINVAL; + + addr = dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base + offset; + + if (is_write) { + if (region->uaccess_write) + ret = region->uaccess_write(kvm_get_vcpu(dev->kvm, 0), + addr, len, *reg); + else + region->write(kvm_get_vcpu(dev->kvm, 0), addr, len, *reg); + } else { + if (region->uaccess_read) + *reg = region->uaccess_read(kvm_get_vcpu(dev->kvm, 0), + addr, len); + else + *reg = region->read(kvm_get_vcpu(dev->kvm, 0), addr, len); + } + + return ret; +}
diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index c10a113..e5657cb 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -2134,9 +2134,9 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u64 val; /* - * MAPC with V=0 keeps the ITEs mapped but drops their collection, - * and with it the ICID. Save a zeroed entry, which the restore path - * reads back as invalid. + * MAPC with V=0 and clearing GITS_BASER<coll>.Valid both keep the + * ITEs mapped while dropping their collection, and with it the ICID. + * Save a zeroed entry, which the restore path reads back as invalid. */ if (!ite->collection) return vgic_its_write_entry_lock(its, gpa, 0ULL, ite);
diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 90be9944..1c205cb 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c
@@ -181,6 +181,14 @@ static int kvm_vgic_addr(struct kvm *kvm, struct kvm_device_attr *attr, bool wri addr |= (u64)rdreg->count << KVM_VGIC_V3_RDIST_COUNT_SHIFT; goto out; } + case KVM_VGIC_V5_ADDR_TYPE_IRS: + r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V5); + if (r) + break; + addr_ptr = &vgic->vgic_v5_irs_data->vgic_v5_irs_base; + alignment = SZ_64K; + size = KVM_VGIC_V5_IRS_SIZE; + break; default: r = -ENODEV; } @@ -224,31 +232,48 @@ static int vgic_set_common_attr(struct kvm_device *dev, if (get_user(val, uaddr)) return -EFAULT; - /* - * We require: - * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs - * - at most 1024 interrupts - * - a multiple of 32 interrupts - */ - if (val < (VGIC_NR_PRIVATE_IRQS + 32) || - val > VGIC_MAX_RESERVED || - (val & 31)) - return -EINVAL; + if (!vgic_is_v5(dev->kvm)) { + /* + * We require: + * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs + * - at most 1024 interrupts + * - a multiple of 32 interrupts + */ + if (val < (VGIC_NR_PRIVATE_IRQS + 32) || + val > VGIC_MAX_RESERVED || (val & 31)) + return -EINVAL; - mutex_lock(&dev->kvm->arch.config_lock); + mutex_lock(&dev->kvm->arch.config_lock); - /* - * Either userspace has already configured NR_IRQS or - * the vgic has already been initialized and vgic_init() - * supplied a default amount of SPIs. - */ - if (dev->kvm->arch.vgic.nr_spis) - ret = -EBUSY; - else - dev->kvm->arch.vgic.nr_spis = - val - VGIC_NR_PRIVATE_IRQS; + /* + * Either userspace has already configured NR_IRQS or + * the vgic has already been initialized and vgic_init() + * supplied a default amount of SPIs. + */ + if (dev->kvm->arch.vgic.nr_spis) + ret = -EBUSY; + else + dev->kvm->arch.vgic.nr_spis = + val - VGIC_NR_PRIVATE_IRQS; - mutex_unlock(&dev->kvm->arch.config_lock); + mutex_unlock(&dev->kvm->arch.config_lock); + } else { + /* + * GICv5 reports a number of SPIs, not a total number of + * interrupts. Require a multiple of 32 SPIs. + */ + if (val < VGIC_V5_DEFAULT_NR_SPIS || + val > VGIC_V5_MAX_NR_SPIS || + (val & 31)) + return -EINVAL; + + mutex_lock(&dev->kvm->arch.config_lock); + if (vgic_initialized(dev->kvm) || dev->kvm->arch.vgic.nr_spis) + ret = -EBUSY; + else + dev->kvm->arch.vgic.nr_spis = val; + mutex_unlock(&dev->kvm->arch.config_lock); + } return ret; } @@ -299,9 +324,14 @@ static int vgic_get_common_attr(struct kvm_device *dev, return (r == -ENODEV) ? -ENXIO : r; case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: { u32 __user *uaddr = (u32 __user *)(long)attr->addr; - - r = put_user(dev->kvm->arch.vgic.nr_spis + - VGIC_NR_PRIVATE_IRQS, uaddr); + /* Older GICs */ + if (!vgic_is_v5(dev->kvm)) { + r = put_user(dev->kvm->arch.vgic.nr_spis + + VGIC_NR_PRIVATE_IRQS, + uaddr); + } else { + r = put_user(dev->kvm->arch.vgic.nr_spis, uaddr); + } break; } } @@ -512,7 +542,7 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, * Allow access to certain ID-like registers prior to VGIC initialization, * thereby allowing the VMM to provision the features / sizing of the VGIC. */ -static bool reg_allowed_pre_init(struct kvm_device_attr *attr) +static bool v3_reg_allowed_pre_init(struct kvm_device_attr *attr) { if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS) return false; @@ -575,7 +605,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev, mutex_lock(&dev->kvm->arch.config_lock); - if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) { + if (!(vgic_initialized(dev->kvm) || v3_reg_allowed_pre_init(attr))) { ret = -EBUSY; goto out; } @@ -743,26 +773,178 @@ static int vgic_v5_get_userspace_ppis(struct kvm_device *dev, return ret; } +int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr) +{ + unsigned long vgic_mpidr, mpidr_reg; + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + vgic_mpidr = (attr->attr & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) >> + KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT; + + mpidr_reg = VGIC_TO_MPIDR(vgic_mpidr); + reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg); + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + reg_attr->vcpu = kvm_get_vcpu(dev->kvm, 0); + break; + default: + return -EINVAL; + } + + if (!reg_attr->vcpu) + return -EINVAL; + + reg_attr->addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK; + + return 0; +} + +/* + * Some registers can potentially be read before the core GIC & IRS has been + * initialised. Right now, everything is required to be post-init. + */ +static bool v5_reg_allowed_pre_init(struct kvm_device_attr *attr) +{ + return false; +} + +static bool vgic_v5_vm_has_run_once(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu_has_run_once(vcpu)) + return true; + } + + return false; +} + +/* + * vgic_v5_attr_regs_access - allows user space to access VGIC v5 state + * + * @dev: kvm device handle + * @attr: kvm device attribute + * @is_write: true if userspace is writing a register + */ +static int vgic_v5_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + bool is_write) +{ + u64 __user *uaddr = (u64 __user *)(unsigned long)attr->addr; + struct vgic_reg_attr reg_attr; + struct kvm_vcpu *vcpu; + bool uaccess; + u64 val; + int ret; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + /* Sysregs uaccess is performed by the sysreg handling code */ + uaccess = false; + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; + default: + uaccess = true; + } + + if (uaccess && is_write) { + if (get_user(val, uaddr)) + return -EFAULT; + } + + mutex_lock(&dev->kvm->lock); + + if (kvm_trylock_all_vcpus(dev->kvm)) { + mutex_unlock(&dev->kvm->lock); + return -EBUSY; + } + + mutex_lock(&dev->kvm->arch.config_lock); + + if (!(vgic_initialized(dev->kvm) || v5_reg_allowed_pre_init(attr))) { + ret = -EBUSY; + goto out; + } + + if (is_write && vgic_v5_vm_has_run_once(dev->kvm)) { + ret = -EBUSY; + goto out; + } + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + ret = vgic_v5_cpu_sysregs_uaccess(vcpu, attr, is_write); + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + /* + * The IRS registers are a mixture of 32-bit and 64-bit + * registers. Internally, we always perform the correctly sized + * access, but the UAPI is defined in such a way that we are + * always provided a __u64 by userspace. When userspace writes, + * the upper 32-bits are ignored for 32-bit accesses, and on a + * read any 32-bit accesses are written back to user memory + * using the full 64-bits. + */ + ret = vgic_v5_irs_attr_regs_access(dev, attr, &val, is_write); + break; + default: + ret = -EINVAL; + break; + } + +out: + mutex_unlock(&dev->kvm->arch.config_lock); + kvm_unlock_all_vcpus(dev->kvm); + mutex_unlock(&dev->kvm->lock); + + if (!ret && uaccess && !is_write) + ret = put_user(val, uaddr); + + return ret; +} + static int vgic_v5_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + if (attr->attr) + return -ENXIO; + + return vgic_v5_irs_restore_ists(dev->kvm, attr); + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_attr_regs_access(dev, attr, true); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: - return -ENXIO; + break; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: - return vgic_set_common_attr(dev, attr); + break; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: default: return -ENXIO; } + break; default: return -ENXIO; } + return vgic_set_common_attr(dev, attr); } static int vgic_v5_get_attr(struct kvm_device *dev, @@ -770,21 +952,33 @@ static int vgic_v5_get_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + if (attr->attr) + return -ENXIO; + + return vgic_v5_irs_save_ists(dev->kvm, attr); + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_attr_regs_access(dev, attr, false); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: - return -ENXIO; + break; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: - return vgic_get_common_attr(dev, attr); + break; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: return vgic_v5_get_userspace_ppis(dev, attr); default: return -ENXIO; } + break; default: return -ENXIO; } + + return vgic_get_common_attr(dev, attr); } static int vgic_v5_has_attr(struct kvm_device *dev, @@ -792,18 +986,30 @@ static int vgic_v5_has_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: - case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + switch (attr->attr) { + case KVM_VGIC_V5_ADDR_TYPE_IRS: + return 0; + } return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_has_attr_regs(dev, attr); + case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + return 0; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: return 0; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: return 0; + case KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES: default: return -ENXIO; } + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + return attr->attr ? -ENXIO : 0; default: return -ENXIO; }
diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 5913a20..5ab789a 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c
@@ -818,16 +818,21 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); rd_dev->redist_vcpu = vcpu; + /* Protected by slots_lock */ + rdreg->free_index++; + mutex_unlock(&kvm->arch.config_lock); ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, 2 * SZ_64K, &rd_dev->dev); - if (ret) - return ret; + if (ret) { + guard(mutex)(&vcpu->kvm->arch.config_lock); + vgic_cpu->rdreg->free_index--; + vgic_cpu->rdreg = NULL; + vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF; + } - /* Protected by slots_lock */ - rdreg->free_index++; - return 0; + return ret; out_unlock: mutex_unlock(&kvm->arch.config_lock); @@ -841,7 +846,8 @@ void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev); } -static int vgic_register_all_redist_iodevs(struct kvm *kvm) +static int vgic_register_all_redist_iodevs(struct kvm *kvm, + struct vgic_redist_region *rdreg) { struct kvm_vcpu *vcpu; unsigned long c; @@ -861,7 +867,14 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm) for (i = 0; i < c; i++) { vcpu = kvm_get_vcpu(kvm, i); + if (vcpu->arch.vgic_cpu.rdreg != rdreg) + continue; + vgic_unregister_redist_iodev(vcpu); + + guard(mutex)(&vcpu->kvm->arch.config_lock); + vcpu->arch.vgic_cpu.rdreg = NULL; + vcpu->arch.vgic_cpu.rd_iodev.base_addr = VGIC_ADDR_UNDEF; } } @@ -970,10 +983,13 @@ void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdre int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) { + struct vgic_redist_region *rdreg; int ret; mutex_lock(&kvm->arch.config_lock); ret = vgic_v3_alloc_redist_region(kvm, index, addr, count); + if (!ret) + rdreg = vgic_v3_rdist_region_from_index(kvm, index); mutex_unlock(&kvm->arch.config_lock); if (ret) return ret; @@ -982,12 +998,9 @@ int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) * Register iodevs for each existing VCPU. Adding more VCPUs * afterwards will register the iodevs when needed. */ - ret = vgic_register_all_redist_iodevs(kvm); + ret = vgic_register_all_redist_iodevs(kvm, rdreg); if (ret) { - struct vgic_redist_region *rdreg; - mutex_lock(&kvm->arch.config_lock); - rdreg = vgic_v3_rdist_region_from_index(kvm, index); vgic_v3_free_redist_region(kvm, rdreg); mutex_unlock(&kvm->arch.config_lock); return ret;
diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index 74d76de..fddb9da 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c
@@ -1065,6 +1065,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, case IODEV_ITS: data = region->its_read(vcpu->kvm, iodev->its, addr, len); break; + case IODEV_GICV5_IRS: + data = region->read(vcpu, addr, len); + break; } vgic_data_host_to_mmio_bus(val, len, data); @@ -1095,6 +1098,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, case IODEV_ITS: region->its_write(vcpu->kvm, iodev->its, addr, len, data); break; + case IODEV_GICV5_IRS: + region->write(vcpu, addr, len, data); + break; } return 0;
diff --git a/arch/arm64/kvm/vgic/vgic-mmio.h b/arch/arm64/kvm/vgic/vgic-mmio.h index 50dc8022..38ed730 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.h +++ b/arch/arm64/kvm/vgic/vgic-mmio.h
@@ -217,6 +217,8 @@ unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev); unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev); +unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev); + u64 vgic_sanitise_outer_cacheability(u64 reg); u64 vgic_sanitise_inner_cacheability(u64 reg); u64 vgic_sanitise_shareability(u64 reg);
diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c new file mode 100644 index 0000000..d577e48 --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c
@@ -0,0 +1,1872 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#include <kvm/arm_vgic.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/mutex.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/xarray.h> +#include <asm/kvm_mmu.h> + +#include "vgic.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_v5_vmt *vmt_info; + +/* Serialises IRS MMIO interaction (commands) and accesses to IRS tables. */ +DEFINE_RAW_SPINLOCK(vgic_v5_irs_lock); + +/* Serialises lazy installation of shared second-level VMTs. */ +static DEFINE_MUTEX(vmt_l2_lock); + +static DEFINE_XARRAY(vm_info); + +/* Level 1 Virtual Machine Table Entry */ +#define GICV5_VMTEL1E_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design */ +#define GICV5_VMTEL1E_L2_ADDR GENMASK(51, 12) + +#define GICV5_VMTEL2E_SIZE 32ULL +/* An L2 table (two-level VMT) is ALWAYS 4kB! */ +#define GICV5_VMT_L2_TABLE_SIZE 4096ULL +#define GICV5_VMT_L2_TABLE_ENTRIES (GICV5_VMT_L2_TABLE_SIZE / GICV5_VMTEL2E_SIZE) + +/* + * As the L2 VMTE is a large data structure, we are splitting it into 4 parts. + * We only mask and shift WITHIN each part for simplicity. + */ +/* First 64-bit chunk */ +#define GICV5_VMTEL2E_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_VMD_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VMD_ADDR GENMASK_ULL(55, 3) +/* Second 64-bit chunk */ +#define GICV5_VMTEL2E_VPET_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VPET_ADDR GENMASK_ULL(55, 3) +#define GICV5_VMTEL2E_VPE_ID_BITS GENMASK_ULL(63, 59) +/* Third & fourth 64-bit chunks (the encodings are the same for each) */ +#define GICV5_VMTEL2E_IST_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_IST_L2SZ GENMASK_ULL(2, 1) +#define GICV5_VMTEL2E_IST_ADDR_SHIFT 6ULL +#define GICV5_VMTEL2E_IST_ADDR GENMASK_ULL(55, 6) +#define GICV5_VMTEL2E_IST_ISTSZ GENMASK_ULL(57, 56) +#define GICV5_VMTEL2E_IST_STRUCTURE BIT_ULL(58) +#define GICV5_VMTEL2E_IST_ID_BITS GENMASK_ULL(63, 59) + +/* Virtual PE Table Entry */ +#define GICV5_VPE_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design. */ +#define GICV5_VPED_ADDR GENMASK_ULL(55, 3) + +/* L2 Interrupt State Table Entry */ +#define GICV5_ISTL2E_PENDING BIT(0) +#define GICV5_ISTL2E_ACTIVE BIT(1) +#define GICV5_ISTL2E_HM BIT(2) +#define GICV5_ISTL2E_ENABLE BIT(3) +#define GICV5_ISTL2E_IRM BIT(4) +#define GICV5_ISTL2E_HWU GENMASK(10, 9) +#define GICV5_ISTL2E_PRIORITY GENMASK(15, 11) +#define GICV5_ISTL2E_IAFFID GENMASK(31, 16) + +#define GICV5_ISTE_SIZE(istsz) BIT((istsz) + 2) +#define GICV5_LINEAR_IST_SIZE(id_bits, istsz) \ + (BIT(id_bits) * GICV5_ISTE_SIZE(istsz)) + +/* + * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of + * the VMTE (0-based). We call this a section here in an attempt to simplify the + * code. + */ +#define GICV5_VMTEL2_LPI_SECTION 2 +#define GICV5_VMTEL2_SPI_SECTION 3 + +struct vgic_v5_ist_desc { + struct vgic_v5_vm_info *vmi; + void *base; + unsigned int id_bits; + unsigned int istsz; + unsigned int l2sz; + size_t iste_size; + bool present; +}; + +struct vgic_v5_two_level_ist_shape { + size_t l1_entries; + size_t l2_entries; +}; + +struct vgic_v5_pending_irq { + u32 irq; + struct list_head next; +}; + +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, + unsigned int istsz); +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2_split); +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_spi_ist_free(struct kvm *kvm); + +/* + * Our IRS might be coherent or non-coherent. If coherent, we can just emit a + * DSB to ensure that we're in sync. However, when non-coherent, we need to + * manage our cached data explicitly. + * + * This helper is used to handle both coherent and non-coherent IRSes, and + * handles all combinations of cleaning and invalidating to the PoC. + */ +static void vgic_v5_clean_inval(void *va, size_t size) +{ + unsigned long base = (unsigned long)va; + + dsb(ishst); + + if (kvm_vgic_global_state.vgic_v5_irs_caps.non_coherent) + dcache_clean_inval_poc(base, base + size); +} + +static void vgic_v5_drain_pending_irqs(struct kvm *kvm, + struct vgic_v5_vm_info *vmi, + bool reinject) +{ + struct vgic_v5_pending_irq *pirq, *tmp; + + list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) { + if (reinject) + kvm_call_hyp(__vgic_v5_vdpend, pirq->irq, true, + vgic_v5_vm_id(kvm)); + + list_del(&pirq->next); + kfree(pirq); + } +} + +/* + * Create a linear VM Table. Directly using the number of entries supplied as + * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per + * the GICv5 requirements for the IRS_VMT_BASER. + */ +static int vgic_v5_alloc_vmt_linear(unsigned int num_entries) +{ + vmt_info->linear.vmt_base = kzalloc_objs(*vmt_info->linear.vmt_base, + num_entries); + if (!vmt_info->linear.vmt_base) + return -ENOMEM; + + vgic_v5_clean_inval(vmt_info->linear.vmt_base, + num_entries * sizeof(struct vmtl2_entry)); + + return 0; +} + +/* + * Allocate the first level of a two-level VM table. The second-level VM tables + * are allocated on demand (by vgic_v5_alloc_l2_vmt()). + */ +static int vgic_v5_alloc_vmt_two_level(unsigned int num_entries) +{ + /* + * Each L2 VMT array is always 4k-sized (covering 128 VMs). This is + * mandated by the GICv5 specification (GICv5 EAC0 Specification rule + * D_LSPBK). Hence, round up the number of entries to be at least 128 + * (or the next highest power of two as we give the HW the number of VM + * ID bits). + */ + if (num_entries < GICV5_VMT_L2_TABLE_ENTRIES) + num_entries = GICV5_VMT_L2_TABLE_ENTRIES; + num_entries = roundup_pow_of_two(num_entries); + + vmt_info->l2.num_l1_ents = (num_entries / GICV5_VMT_L2_TABLE_ENTRIES); + vmt_info->l2.vmt_base = kzalloc_objs(*vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents); + if (!vmt_info->l2.vmt_base) + return -ENOMEM; + + vmt_info->l2.l2ptrs = kzalloc_objs(*vmt_info->l2.l2ptrs, + vmt_info->l2.num_l1_ents, + GFP_KERNEL); + if (!vmt_info->l2.l2ptrs) { + kfree(vmt_info->l2.vmt_base); + return -ENOMEM; + } + + vgic_v5_clean_inval(vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents * sizeof(vmtl1_entry)); + + return 0; +} + +/* + * Allocate a second level VMT, if required. This can be called eagerly, and + * will only perform the allocation if required. + */ +static int vgic_v5_alloc_l2_vmt(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd = VMT_L2_MAP; + struct vmtl2_entry *l2_table; + unsigned int l1_index; + int ret; + + /* Nothing to do if we have linear tables! */ + if (!vmt_info->two_level) + return 0; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return -EINVAL; + + /* + * We have 4k-sized L2 tables - this is mandated by the spec for + * two-level VMTs (GICv5 EAC0 Specification rule D_LSPBK). This means + * that we have 128 entries per L1 VMTE. + */ + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + + guard(mutex)(&vmt_l2_lock); + + /* Already valid? Great! */ + if (vmt_info->l2.l2ptrs[l1_index]) + return 0; + + l2_table = kzalloc_objs(*l2_table, GICV5_VMT_L2_TABLE_ENTRIES); + if (!l2_table) + return -ENOMEM; + + /* The VMT is shared between all VMs. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(l2_table, GICV5_VMT_L2_TABLE_SIZE); + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], + cpu_to_le64(virt_to_phys(l2_table))); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + } + + /* + * VMAP in the L2 VMT via the IRS. We use any of the VM's CPUs as a + * conduit for interacting with the host's IRS. In the current case, + * this lets us resolve the VM ID to pass to the hardware. + */ + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + + /* We've failed to make the L2 VMT valid - things are very broken! */ + if (ret) { + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Remove the pointer from L1 table */ + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], 0); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + } + + kfree(l2_table); + return ret; + } + + vmt_info->l2.l2ptrs[l1_index] = l2_table; + + return 0; +} + +/* + * Allocate the top-level VMT. This can either be linear or two-level. + */ +int vgic_v5_vmt_allocate(unsigned int max_vpes) +{ + int ret; + + /* Allocate the tracking structure */ + vmt_info = kzalloc_obj(*vmt_info, GFP_KERNEL); + if (!vmt_info) + return -ENOMEM; + + ida_init(&vmt_info->vm_id_ida); + vmt_info->max_vpes = max_vpes; + vmt_info->vmd_size = vgic_v5_irs_vmd_size(&irs_caps); + vmt_info->vped_size = vgic_v5_irs_vped_size(&irs_caps); + vmt_info->two_level = vgic_v5_irs_two_level_vmt_support(&irs_caps); + vmt_info->num_entries = vgic_v5_irs_max_vms(&irs_caps); + + if (vmt_info->two_level) + ret = vgic_v5_alloc_vmt_two_level(vmt_info->num_entries); + else + ret = vgic_v5_alloc_vmt_linear(vmt_info->num_entries); + + /* If anything failed, free our tracking structure before returning */ + if (ret) { + kfree(vmt_info); + vmt_info = NULL; + } + + return ret; +} + +/* + * Free the VMT and associated tracking structures. This isn't strictly expected + * to be called in general operation, but instead exists for completeness. + */ +int vgic_v5_vmt_free(void) +{ + if (!vmt_info) + return 0; + + if (!vmt_info->two_level) { + kfree(vmt_info->linear.vmt_base); + } else { + /* Free the L2 tables; kfree(NULL) is safe */ + for (int i = 0; i < vmt_info->l2.num_l1_ents; ++i) + kfree(vmt_info->l2.l2ptrs[i]); + kfree(vmt_info->l2.l2ptrs); + + /* And now free the L1 table */ + kfree(vmt_info->l2.vmt_base); + } + + ida_destroy(&vmt_info->vm_id_ida); + kfree(vmt_info); + vmt_info = NULL; + + return 0; +} + +/* + * Look up a VMT Entry by VM ID. + */ +static struct vmtl2_entry *vgic_v5_get_l2_vmte(u32 vm_id) +{ + unsigned int l1_index, l2_index; + struct vmtl2_entry *l2_table; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return ERR_PTR(-EINVAL); + + if (!vmt_info->two_level) + return &vmt_info->linear.vmt_base[vm_id]; + + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + l2_index = vm_id % GICV5_VMT_L2_TABLE_ENTRIES; + + if (l1_index >= vmt_info->l2.num_l1_ents) + return ERR_PTR(-E2BIG); + + if (!vmt_info->l2.l2ptrs[l1_index]) + return ERR_PTR(-EINVAL); + + l2_table = vmt_info->l2.l2ptrs[l1_index]; + return &l2_table[l2_index]; +} + +/* + * Zero a VMT Entry, and flush & invalidate to the PoC, if required. + */ +static int vgic_v5_reset_vmte(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* + * The VMT is normal memory shared with the IRS. Invalidate before + * rewriting the entry so that cacheline-granular maintenance cannot + * later push stale data for neighbouring IRS-visible state back to + * memory. + */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + + /* + * Prevent the compiler from eliding the individual VMTE + * stores. Ordering and visibility to the IRS are provided by the + * surrounding cache maintenance and command protocol, not by + * WRITE_ONCE(). + * + * The same compiler-access constraint applies to READ_ONCE() users in + * this file: when inspecting IRS-visible table entries, read the field + * exactly once and prevent the compiler from reusing, merging or + * tearing the access. Coherency and freshness for non-coherent IRSes + * still come from the surrounding cache maintenance. + */ + WRITE_ONCE(vmte->val[0], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[2], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[3], cpu_to_le64(0ULL)); + + /* And make our write visible to the IRS (if non-coherent) */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Use the IDA to allocate a new VM ID, and track it in the gicv5_vm data + * structure. If we're out of VM IDs, the IDA catches that, and we return the + * error (-ENOSPC). If we've previously allocated a VM ID, we catch that too and + * return -EBUSY. + */ +int vgic_v5_allocate_vm_id(struct kvm *kvm) +{ + int id; + + if (kvm->arch.vgic.gicv5_vm.vm_id != VGIC_V5_VM_ID_INVAL) + return -EBUSY; + + id = ida_alloc_max(&vmt_info->vm_id_ida, vmt_info->num_entries - 1u, + GFP_KERNEL); + if (id < 0) + return id; + + kvm->arch.vgic.gicv5_vm.vm_id = id; + + return 0; +} + +/* + * Release the VM ID to allow it to be reallocated in the future. + */ +void vgic_v5_release_vm_id(struct kvm *kvm) +{ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + ida_free(&vmt_info->vm_id_ida, kvm->arch.vgic.gicv5_vm.vm_id); + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; +} + +/* + * Initialise an entry in the VMT based on the index of the VM. + * + * Note: We don't mark the VMTE as valid as this needs to be done by + * the hardware. + */ +int vgic_v5_vmte_init(struct kvm *kvm) +{ + size_t vmd_alloc_size, vpet_alloc_size, vped_alloc_size; + void *vped_base = NULL, *vmd = NULL; + struct vgic_v5_vm_info *vmi = NULL; + u64 tmp, vmte_val0 = 0, vmte_val1; + u32 vm_id = vgic_v5_vm_id(kvm); + int ret, nr_cpus, nr_vcpus; + bool vmi_inserted = false; + struct vmtl2_entry *vmte; + vpe_entry *vpet = NULL; + struct kvm_vcpu *vcpu; + u16 max_vpe_id = 0; + unsigned long i; + + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus > vmt_info->max_vpes) + return -E2BIG; + + /* + * If we're using two-level VMTs, L2 is allocated on demand. For linear + * VMTs, this is a NOP. + */ + ret = vgic_v5_alloc_l2_vmt(kvm); + if (ret) + return ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* If the entry is already valid, something went wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + vmi = kzalloc_obj(*vmi); + if (!vmi) { + ret = -ENOMEM; + goto out_fail; + } + + ret = xa_insert(&vm_info, vm_id, vmi, GFP_KERNEL); + if (ret) + goto out_fail; + vmi_inserted = true; + + /* + * If we are restoring the state of a guest, we need to re-inject any + * IRQs that were pending when the state of the guest was originally + * saved. We use the pending_irqs list for this. + */ + INIT_LIST_HEAD(&vmi->pending_irqs); + + /* Allocate and assign the VM Descriptor, if required. */ + if (vmt_info->vmd_size != 0) { + vmd_alloc_size = round_up(vmt_info->vmd_size, + dma_get_cache_alignment()); + vmd = kzalloc(vmd_alloc_size, GFP_KERNEL); + if (!vmd) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vmd_base = vmd; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VMD_ADDR, + virt_to_phys(vmd) >> GICV5_VMTEL2E_VMD_ADDR_SHIFT); + vmte_val0 = tmp; + } + + /* + * Allocate and assign the VPE Table. + * + * First of all, iterate over all vcpus to find the highest VPE ID we + * require - we need to ensure that we have enough storage for all + * vcpu_id values that userspace has picked and not just the total + * number of vcpus. This gives us the number of VPEs required for the + * VM. + * + * Round up the number of VPEs to a whole power of two as we cannot + * describe non-powers-of-two in the VMTE field as it conveys the number + * of ID bits used and not the number of vPEs. IRS_IDR1.IAFFID_BITS is + * encoded as N - 1, so expose at least one VPE ID bit even for a + * single-vCPU VM to keep the views consistent. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + u16 vpe_id = vgic_v5_vpe_id(vcpu); + + if (vpe_id > max_vpe_id) + max_vpe_id = vpe_id; + } + + nr_cpus = max(2UL, roundup_pow_of_two(max_vpe_id + 1)); + vmi->vpe_id_bits = fls(nr_cpus) - 1; + + vpet_alloc_size = round_up((size_t)nr_cpus * sizeof(*vpet), + dma_get_cache_alignment()); + vpet = kzalloc(vpet_alloc_size, GFP_KERNEL); + if (!vpet) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vpet_base = vpet; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VPET_ADDR, + virt_to_phys(vpet) >> GICV5_VMTEL2E_VPET_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_VPE_ID_BITS, vmi->vpe_id_bits); + vmte_val1 = tmp; + + /* + * Allocate a dense VPED array indexed by vcpu_idx. The VPET is indexed + * by the potentially sparse vcpu_id, but using that ID here would waste + * memory. Given that this is not userspace visible, we can cheat a bit + * and use the dense index instead. This is the ONLY place that we do + * this. + * + * Round the requested size up to a whole cacheline. kzalloc() gurantees + * natural alignment, so we ensure that the cachelines cannot be shared + * with unrelated slab objects. VPED and cacheline sizes are powers of + * two, so this also preserves the required VPED alignment when a VPED + * is larger than a cacheline. + */ + vped_alloc_size = round_up((size_t)nr_vcpus * vmt_info->vped_size, + dma_get_cache_alignment()); + vped_base = kzalloc(vped_alloc_size, GFP_KERNEL); + if (!vped_base) { + ret = -ENOMEM; + goto out_fail; + } + vmi->vped_base = vped_base; + + if (vmd) + vgic_v5_clean_inval(vmd, vmd_alloc_size); + vgic_v5_clean_inval(vpet, vpet_alloc_size); + vgic_v5_clean_inval(vped_base, vped_alloc_size); + + /* Publish the VMTE while serialising access to the shared VMT. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[0], cpu_to_le64(vmte_val0)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(vmte_val1)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + kvm->arch.vgic.gicv5_vm.vmte_allocated = true; + + return 0; + +out_fail: + /* kfree(NULL) is safe so we can just kfree() at leisure */ + kfree(vmd); + kfree(vpet); + kfree(vped_base); + if (vmi_inserted) + xa_erase(&vm_info, vm_id); + kfree(vmi); + + vgic_v5_reset_vmte(kvm); + + return ret; +} + +/* + * Release the VMT Entry, freeing up any allocated data structures before + * zeroing the VMTE. + * + * The VMTE must be marked as invalid before it is released. + */ +int vgic_v5_vmte_release(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + int ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* Reject if the VMTE has not been marked as invalid! */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + goto no_vmi; + + /* Unlikely, but possible. Avoid leaking the memory. */ + vgic_v5_drain_pending_irqs(kvm, vmi, false); + + /* If we have an LPI IST, free it */ + if (vmi->h_lpi_ist) { + ret = vgic_v5_lpi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_lpi_ist = NULL; + + /* If we have an SPI IST, free it */ + if (vmi->h_spi_ist) { + ret = vgic_v5_spi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_spi_ist = NULL; + + kfree(vmi->vped_base); + kfree(vmi->vpet_base); + kfree(vmi->vmd_base); + + xa_erase(&vm_info, vm_id); + kfree(vmi); + +no_vmi: + /* + * If we didn't get far enough into allocating a VMTE to create the VM + * info structure, then we just zero the VMTE and move on. There's + * nothing else we can realistically do here. + */ + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + kvm->arch.vgic.gicv5_vm.vmte_allocated = false; + + return 0; +} + +/* + * Provide a way for the IRS MMIO emulation to correctly populate the number of + * IAFFID bits (which correspond to our vpe_id_bits. + */ +u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return 0; + + return vmi->vpe_id_bits; +} + +/* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */ +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + vpe_entry *vpet_base; + void *vped; + + /* Make sure we're not over what the hardware supports */ + if (vpe_id >= vmt_info->max_vpes) + return -E2BIG; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + + /* If the VPETE for this CPU is already valid we've gone wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(&vpet_base[vpe_id], sizeof(*vpet_base)); + if (le64_to_cpu(READ_ONCE(vpet_base[vpe_id])) & GICV5_VPE_VALID) + return -EBUSY; + } + + vped = (u8 *)vmi->vped_base + + (size_t)vcpu->vcpu_idx * vmt_info->vped_size; + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + phys_addr_t paddr = virt_to_phys(vped) & GICV5_VPED_ADDR; + WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(paddr)); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} + +/* Clear the VPE's table entry after the VMTE has been made invalid. */ +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + vpe_entry *vpet_base; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EBUSY; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vpet_base[vpe_id], 0ULL); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} + +phys_addr_t vgic_v5_get_vmt_base(void) +{ + phys_addr_t vmt_base; + + if (!vmt_info->two_level) + vmt_base = virt_to_phys(vmt_info->linear.vmt_base); + else + vmt_base = virt_to_phys(vmt_info->l2.vmt_base); + + return vmt_base; +} + +/* + * Assign an already allocated IST to the VM by populating the fields in the + * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even + * if the paths to reach it might be vastly different. + */ +static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base, + bool two_level, unsigned int id_bits, + unsigned int l2sz, unsigned int istsz, + bool spi_ist) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd; + struct vmtl2_entry *vmte; + unsigned int section; + u64 tmp; + int ret; + + /* + * The L2 VMTE comprises four 64-bit "sections", where sections 2 & 3 + * describe the LPI and SPI ISTs, respectively. Both the LPI and SPI + * sections have the same layout, and as we are either operating on SPIs + * or LPIs we pick a section of the VMTE to modify up-front. + * + * See the GICv5 EAC0 Specification 11.2.2 for more details about the + * VMTE layout. + */ + section = spi_ist ? GICV5_VMTEL2_SPI_SECTION : GICV5_VMTEL2_LPI_SECTION; + + if (ist_base & ~GICV5_VMTEL2E_IST_ADDR) { + pr_err_ratelimited("kvm [%i]: IST misaligned: address 0x%llx, mask 0x%llx\n", + task_pid_nr(current), ist_base, + GICV5_VMTEL2E_IST_ADDR); + return -EINVAL; + } + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + tmp = FIELD_PREP(GICV5_VMTEL2E_IST_L2SZ, l2sz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ADDR, + ist_base >> GICV5_VMTEL2E_IST_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ISTSZ, istsz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ID_BITS, id_bits); + if (two_level) + tmp |= GICV5_VMTEL2E_IST_STRUCTURE; + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Bail if already allocated */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[section])) & + GICV5_VMTEL2E_IST_VALID) + return -EINVAL; + + WRITE_ONCE(vmte->val[section], cpu_to_le64(tmp)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + /* Finally, mark the entry as valid */ + cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID; + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + if (ret) { + /* + * A timeout does not tell us whether the IRS consumed the + * command, so we're in some indeterminate and potentially + * transient state. It may still make the VIST valid and access + * the backing memory, so leave both the VMTE and allocation + * intact. Killing the VM ensures that teardown is the only path + * that can reclaim them, after it has successfully made the + * complete VMTE invalid. + */ + if (ret == -ETIMEDOUT) { + kvm_vm_dead(kvm); + return ret; + } + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], 0ULL); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return ret; + } + + return 0; +} + +/* + * Allocate a Linear IST - always used for SPIs and potentially LPIs. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz is the FIELD used by GICv5, not the actual size (or log2() of the + * size). + */ +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, unsigned int istsz) +{ + const size_t n = max(5, id_bits + 1 + istsz); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + u32 l1sz; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + /* + * Allocate the IST. We only have one level, so we just use the L2 ISTE. + */ + l1sz = BIT(n + 1); + ist = kzalloc(l1sz, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + if (spi_ist) { + vmi->h_spi_ist = ist; + } else { + vmi->h_lpi_ist_structure = false; + vmi->h_lpi_ist = ist; + } + + vgic_v5_clean_inval(ist, l1sz); + + return 0; +} + +/* + * Allocate the first level of a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + u32 vm_id = vgic_v5_vm_id(kvm); + const u32 l1_size = BIT(n + 1); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + ist = kzalloc(l1_size, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + vmi->h_lpi_ist_structure = true; + vmi->h_lpi_ist = ist; + + vgic_v5_clean_inval(ist, l1_size); + + return 0; +} + +/* + * Allocate ALL of the second level ISTs for a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. The l2_size calculation is from section 11.2.3 of the same document. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + const size_t l2_size = BIT(11 + (2 * l2sz) + 1); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *l2ist; + __le64 *l1ist; + int index; + u64 val; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + l1ist = vmi->h_lpi_ist; + + /* + * Allocate the storage for the pointers to the L2 ISTs (used when + * freeing later). + */ + vmi->h_lpi_l2_ists = kzalloc_objs(*vmi->h_lpi_l2_ists, l1_entries, + GFP_KERNEL_ACCOUNT); + if (!vmi->h_lpi_l2_ists) + return -ENOMEM; + + /* Allocate the L2 IST for each L1 IST entry */ + for (index = 0; index < l1_entries; ++index) { + l2ist = kzalloc(l2_size, GFP_KERNEL_ACCOUNT); + if (!l2ist) { + while (--index >= 0) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + + return -ENOMEM; + } + + /* + * We are not doing on-demand allocation of the L2 ISTs, and are + * instead provisioning the whole IST up front. This means that + * we are able to mark the L2 ISTs as valid in the L1 ISTEs as + * the overall IST is not yet valid. + */ + val = (virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK) | + GICV5_ISTL1E_VALID; + l1ist[index] = cpu_to_le64(val); + + vmi->h_lpi_l2_ists[index] = l2ist; + + vgic_v5_clean_inval(l2ist, l2_size); + } + + /* Handle CMOs for the whole L1 IST in one go */ + vgic_v5_clean_inval(l1ist, l1_entries * sizeof(*l1ist)); + + return 0; +} + +/* Allocate a two-level IST - LPIs, only */ +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + int ret; + + /* + * Allocate the L1 IST first, then all of the L2s. Everything + * is preallocated and we do no on-demand IST allocation. This + * is to avoid needing to track if and when the guest is doing + * on-demand IST allocation. + */ + ret = vgic_v5_alloc_l1_ist(kvm, id_bits, istsz, l2sz); + if (ret) + return ret; + + ret = vgic_v5_alloc_l2_ists(kvm, id_bits, istsz, l2sz); + if (ret) { + /* Free the L1 IST again */ + vmi = xa_load(&vm_info, vm_id); + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; + + return ret; + } + + return 0; +} + +static void vgic_v5_free_allocated_lpi_ist(struct vgic_v5_vm_info *vmi, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2sz) +{ + if (!vmi->h_lpi_ist_structure) { + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; + return; + } + + if (vmi->h_lpi_l2_ists) { + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + int index; + + for (index = 0; index < l1_entries; ++index) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + } + + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; +} + +static void vgic_v5_free_allocated_spi_ist(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return; + + kfree(vmi->h_spi_ist); + vmi->h_spi_ist = NULL; +} + +/* + * Free a Linear IST. Can only happen once the VM is dead. + */ +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + struct vgic_v5_vm_info *vmi; + int section; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + if (spi) { + section = GICV5_VMTEL2_SPI_SECTION; + vgic_v5_free_allocated_spi_ist(kvm); + } else { + section = GICV5_VMTEL2_LPI_SECTION; + vgic_v5_free_allocated_lpi_ist(vmi, 0, 0, 0); + } + + /* The VM should be dead here, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Free a Two-Level IST. Can only happen once the VM is dead. + */ +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi) +{ + unsigned int id_bits, istsz, l2sz; + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + u64 tmp; + int section; + + /* We don't create two-level SPI ISTs, so freeing is a bad idea! */ + if (spi) + return -EINVAL; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + section = GICV5_VMTEL2_LPI_SECTION; + + if (!vmi->h_lpi_ist_structure) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + tmp = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp); + istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp); + l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp); + + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + /* The VM must be dead, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* Helper to determine ISTE size based on metadata requirements */ +static unsigned int vgic_v5_ist_istsz(unsigned int id_bits) +{ + if (!vgic_v5_irs_istmd(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (id_bits >= vgic_v5_irs_istmd_sz(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +/* + * Allocate an IST for SPIs. + * + * We don't anticipate a large number of SPIs being allocated. Therefore, we + * always allocate a Linear IST for SPIs. This will need to be revisited should + * that assumption no longer hold. + */ +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + phys_addr_t base_addr; + unsigned int istsz; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + istsz = vgic_v5_ist_istsz(id_bits); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_spi_ist) + return -EBUSY; + + ret = vgic_v5_alloc_linear_ist(kvm, true, id_bits, istsz); + if (ret) + return ret; + base_addr = virt_to_phys(vmi->h_spi_ist); + + ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz, + true); + if (ret) { + if (ret != -ETIMEDOUT) + vgic_v5_free_allocated_spi_ist(kvm); + return ret; + } + + return 0; +} + +/* + * Free the IST for SPIs. Should only happen once the VM is dead. + */ +static int vgic_v5_spi_ist_free(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->arch.config_lock); + + return vgic_v5_linear_ist_free(kvm, true); +} + +int vgic_v5_lpi_ist_exists(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + return !!vmi->h_lpi_ist; +} + +/* + * Allocate an IST for LPIs. + * + * Unlike with SPIs, we anticipate that the guest will allocate a relatively + * large number of LPIs. Therefore, while we support doing a linear LPI IST, it + * is expected that LPI ISTs will be two-level. + */ +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + unsigned int istsz, l2sz; + phys_addr_t phys_addr; + bool two_level; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_lpi_ist) + return -EBUSY; + + istsz = vgic_v5_ist_istsz(id_bits); + l2sz = gicv5_irs_l2_sz(vgic_v5_irs_ist_l2sz(&irs_caps)); + + /* + * Determine if we want to create a Linear or a Two-Level IST. + * + * A two-level IST is only required when a single L2 IST cannot cover + * the requested ID space. This depends on the L2 IST size selected for + * the IRS, not PAGE_SIZE. Using PAGE_SIZE here would switch to + * two-level too early when the selected L2 IST is larger than a page, + * and the allocation sizing arithmetic would underflow. + */ + two_level = vgic_v5_irs_ist_levels(&irs_caps) && + id_bits > ((10 - istsz) + (2 * l2sz)); + + if (!two_level) + ret = vgic_v5_alloc_linear_ist(kvm, false /* LPIs, not SPIs */, + id_bits, istsz); + else + ret = vgic_v5_alloc_two_level_lpi_ist(kvm, id_bits, istsz, + l2sz); + + if (ret) + return ret; + + phys_addr = virt_to_phys(vmi->h_lpi_ist); + ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz, + istsz, false); + if (ret && ret != -ETIMEDOUT) + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + return ret; +} + +/* Free the LPI IST again */ +int vgic_v5_lpi_ist_free(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + if (!vmi->h_lpi_ist_structure) + return vgic_v5_linear_ist_free(kvm, false); + else + return vgic_v5_two_level_ist_free(kvm, false); +} + +static struct vgic_v5_two_level_ist_shape +vgic_v5_two_level_ist_shape(const struct vgic_v5_ist_desc *ist) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t l2bits, n; + + l2bits = (10 - ist->istsz) + (2 * ist->l2sz); + n = max(2, ist->id_bits - l2bits + 3 - 1); + + shape.l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + shape.l2_entries = BIT(l2bits); + + return shape; +} + +static int vgic_v5_read_vm_ist_desc(struct kvm *kvm, unsigned int section, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + u64 vmte_ist_section; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + vmte_ist_section = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + ist->id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, vmte_ist_section); + ist->istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, vmte_ist_section); + ist->l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, vmte_ist_section); + ist->iste_size = GICV5_ISTE_SIZE(ist->istsz); + + return !!(vmte_ist_section & GICV5_VMTEL2E_IST_VALID); +} + +static int vgic_v5_get_spi_ist_desc(struct kvm *kvm, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + int ret; + + memset(ist, 0, sizeof(*ist)); + + ist->vmi = xa_load(&vm_info, vm_id); + if (!ist->vmi) + return -ENXIO; + + ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_SPI_SECTION, ist); + if (ret < 0) + return ret; + + ist->base = ist->vmi->h_spi_ist; + if (!ret || !ist->base) + return -ENXIO; + + ist->present = true; + return 0; +} + +static int vgic_v5_get_lpi_ist_desc(struct kvm *kvm, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + bool guest_valid, host_valid; + int ret; + + memset(ist, 0, sizeof(*ist)); + + ist->vmi = xa_load(&vm_info, vm_id); + if (WARN_ON_ONCE(!ist->vmi)) + return -ENXIO; + + ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_LPI_SECTION, ist); + if (ret < 0) + return ret; + + host_valid = ret; + guest_valid = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid; + ist->base = ist->vmi->h_lpi_ist; + + /* If there is no IST to save/restore, return without error. */ + if (!guest_valid && !host_valid && !ist->base) + return 0; + + /* Mismatched combination of valid state */ + if (!guest_valid || !host_valid || !ist->base) + return -ENXIO; + + if (ist->vmi->h_lpi_ist_structure && !ist->vmi->h_lpi_l2_ists) + return -ENXIO; + + ist->present = true; + return 0; +} + +/* + * Save a linear host IST to userspace memory. + * + * Only the architected 32-bit ISTE state is stored. Metadata is skipped when + * striding through the host IST. + */ +static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, size_t nr_entries) +{ + size_t index; + int ret; + + vgic_v5_clean_inval(ist->base, + GICV5_LINEAR_IST_SIZE(ist->id_bits, ist->istsz)); + + for (index = 0; index < nr_entries; index++) { + __le32 *h_iste_addr = ist->base + index * ist->iste_size; + u32 h_iste; + + h_iste = le32_to_cpu(READ_ONCE(*h_iste_addr)); + ret = put_user(h_iste, uaddr); + if (ret) + return ret; + + uaddr++; + } + + return 0; +} + +/* + * Save a two-level host IST to userspace memory. + * + * Only the architected 32-bit ISTE state is stored. Metadata is skipped when + * striding through the host IST. + */ +static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t h_l1_index, h_l2_index; + void *h_l2_ist_base; + int ret; + + shape = vgic_v5_two_level_ist_shape(ist); + + vgic_v5_clean_inval(ist->base, + shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist)); + + for (h_l1_index = 0; h_l1_index < shape.l1_entries; h_l1_index++) { + u64 l1_iste; + + /* + * Host L2 ISTs are preallocated. Any invalid L1 entry means the + * host IST state is inconsistent. + */ + l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index])); + if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste)) + return -ENXIO; + + h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index]; + if (!h_l2_ist_base) + return -ENXIO; + + vgic_v5_clean_inval(h_l2_ist_base, + shape.l2_entries * ist->iste_size); + + for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { + void *h_iste_addr = h_l2_ist_base + h_l2_index * ist->iste_size; + u32 h_iste = le32_to_cpu(*(__le32 *)h_iste_addr); + + ret = put_user(h_iste, uaddr); + if (ret) + return ret; + + uaddr++; + } + } + + return 0; +} + +/* + * Save the SPI IST to userspace-provided memory. + */ +int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + struct vgic_v5_ist_desc ist; + u32 __user *uaddr; + int ret; + + ret = vgic_v5_get_spi_ist_desc(kvm, &ist); + if (ret) + return ret; + + uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr; + + /* The host SPI IST is always linear. */ + return vgic_v5_save_linear_ist(&ist, uaddr, + kvm->arch.vgic.nr_spis); +} + +/* + * Save the LPI IST to userspace memory. + * + * The LPI IST may be linear or two-level, so host iteration depends on the + * allocated host shape. + */ +int vgic_v5_save_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + struct vgic_v5_ist_desc ist; + u32 __user *uaddr; + int ret; + + ret = vgic_v5_get_lpi_ist_desc(kvm, &ist); + if (ret) + return ret; + + if (!ist.present) + return 0; + + /* + * Userspace sized the buffer from the guest-visible configuration. + * Refuse to walk a host IST with a different number of entries. + */ + if (ist.id_bits != + kvm->arch.vgic.vgic_v5_irs_data->ist_cfgr.lpi_id_bits) + return -EINVAL; + + uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr; + + if (!ist.vmi->h_lpi_ist_structure) + return vgic_v5_save_linear_ist(&ist, uaddr, + BIT(ist.id_bits)); + + return vgic_v5_save_two_level_ist(&ist, uaddr); +} + +/* + * Track any SPIs and LPIs that were marked as pending at the point where the + * IST was restored. + * + * Restored pending state is cleared from the host ISTE and replayed with VDPEND + * before the VM first runs. + */ +static int vgic_v5_track_pending_irq(struct list_head *pending_irqs, u32 intid, + u32 type) +{ + struct vgic_v5_pending_irq *pirq; + + pirq = kzalloc_obj(*pirq, GFP_KERNEL); + if (!pirq) + return -ENOMEM; + + /* Encode the interrupt as a GICv5 IntID. */ + pirq->irq = FIELD_PREP(GICV5_HWIRQ_TYPE, type) | + FIELD_PREP(GICV5_HWIRQ_ID, intid); + + INIT_LIST_HEAD(&pirq->next); + list_add_tail(&pirq->next, pending_irqs); + + return 0; +} + +/* + * Process and sanitise each restored ISTE. + * + * HWU is for hardware use and must not survive migration. Pending state is + * tracked, cleared from the ISTE, and replayed before the VM first runs. + */ +static int vgic_v5_process_iste(__le32 *iste, struct list_head *pending_irqs, + u32 intid, u32 type) +{ + u32 iste_data = le32_to_cpu(READ_ONCE(*iste)); + int ret; + + /* Pending state is replayed later with VDPEND. */ + if (iste_data & GICV5_ISTL2E_PENDING) { + ret = vgic_v5_track_pending_irq(pending_irqs, intid, type); + if (ret) + return ret; + } + + iste_data &= ~GICV5_ISTL2E_PENDING; + iste_data &= ~GICV5_ISTL2E_HWU; + + WRITE_ONCE(*iste, cpu_to_le32(iste_data)); + + return 0; +} + +static void vgic_v5_restore_spi_config(struct kvm *kvm, __le32 iste, u32 spi) +{ + u32 iste_data = le32_to_cpu(iste); + bool pending = iste_data & GICV5_ISTL2E_PENDING; + struct vgic_irq *irq; + unsigned long flags; + + irq = vgic_get_irq(kvm, vgic_v5_make_spi(spi)); + if (WARN_ON_ONCE(!irq)) + return; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (iste_data & GICV5_ISTL2E_HM) + irq->config = VGIC_CONFIG_LEVEL; + else + irq->config = VGIC_CONFIG_EDGE; + + if (irq->config == VGIC_CONFIG_EDGE) + irq->pending_latch = pending; + else if (pending) + irq->pending_latch = true; + else if (!irq->active) + irq->pending_latch = false; + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(kvm, irq); +} + +static int vgic_v5_restore_ist_entry(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + void *h_iste_addr, __le32 h_iste, + u32 intid, u32 intid_type) +{ + __le32 raw_iste = h_iste; + int ret; + + /* + * Sanitise the IST, clearing HWU & pending fields. Pending state is + * later replayed via GIC VDPEND. + */ + ret = vgic_v5_process_iste(&h_iste, &ist->vmi->pending_irqs, + intid, intid_type); + if (ret) + return ret; + + if (intid_type == GICV5_HWIRQ_TYPE_SPI) + vgic_v5_restore_spi_config(kvm, raw_iste, intid); + + /* + * Zero the full ISTE (incl metadata), and write back the non-metadata + * region, only. + */ + memset(h_iste_addr, 0, ist->iste_size); + WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste); + vgic_v5_clean_inval(h_iste_addr, ist->iste_size); + + return 0; +} + +/* + * Restore a userspace IST image to a linear host IST. + * + * The userspace IST image is a linear array of 32-bit ISTEs. + */ +static int vgic_v5_restore_linear_ist(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, size_t nr_entries, + u32 intid_type) +{ + size_t index; + int ret; + + for (index = 0; index < nr_entries; index++) { + void *h_iste_addr = ist->base + index * ist->iste_size; + u32 h_iste; + + ret = get_user(h_iste, uaddr); + if (ret) + return ret; + + ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, + cpu_to_le32(h_iste), index, intid_type); + if (ret) + return ret; + + uaddr++; + } + + return 0; +} + +/* + * Restore a userspace IST image to a two-level host IST. + * + * The userspace IST image is a linear array of 32-bit ISTEs. + */ +static int vgic_v5_restore_two_level_ist(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, u32 intid_type) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t h_l1_index, h_l2_index; + void *h_l2_ist_base; + int ret; + + shape = vgic_v5_two_level_ist_shape(ist); + + vgic_v5_clean_inval(ist->vmi->h_lpi_ist, + shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist)); + + for (h_l1_index = 0; h_l1_index < shape.l1_entries; ++h_l1_index) { + u64 l1_iste; + + /* + * Host L2 ISTs are preallocated. Any invalid L1 entry means the + * host IST state is inconsistent. + */ + l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index])); + if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste)) + return -ENXIO; + + h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index]; + if (!h_l2_ist_base) + return -ENXIO; + + for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { + void *h_iste_addr = h_l2_ist_base + + h_l2_index * ist->iste_size; + u32 intid = h_l1_index * shape.l2_entries + h_l2_index; + u32 h_iste; + + ret = get_user(h_iste, uaddr); + if (ret) + return ret; + + ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, + cpu_to_le32(h_iste), intid, + intid_type); + if (ret) + return ret; + + uaddr++; + } + } + + return 0; +} + +/* + * Restore the SPI IST from userspace-provided buffer to the host-allocated IST. + */ +int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + u32 __user *uaddr; + struct vgic_v5_ist_desc ist; + int ret; + + ret = vgic_v5_get_spi_ist_desc(kvm, &ist); + if (ret) + return ret; + + uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr; + + /* The host SPI IST is always linear. */ + return vgic_v5_restore_linear_ist(kvm, &ist, uaddr, + kvm->arch.vgic.nr_spis, + GICV5_HWIRQ_TYPE_SPI); +} + +/* + * Restore the LPI IST from userspace memory to the host-allocated LPI IST. + * + * The host LPI IST may be linear or two-level, so host iteration depends on the + * host IST's shape. + */ +int vgic_v5_restore_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + u32 __user *uaddr; + struct vgic_v5_ist_desc ist; + int ret; + + ret = vgic_v5_get_lpi_ist_desc(kvm, &ist); + if (ret) + return ret; + + if (!ist.present) + return 0; + + uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr; + + if (!ist.vmi->h_lpi_ist_structure) + return vgic_v5_restore_linear_ist(kvm, &ist, uaddr, + BIT(ist.id_bits), + GICV5_HWIRQ_TYPE_LPI); + + return vgic_v5_restore_two_level_ist(kvm, &ist, uaddr, + GICV5_HWIRQ_TYPE_LPI); +} + +/* + * Process the pending IRQs removing them from the list and optionally injecting + * them. + */ +static int vgic_v5_process_pending_irqs(struct kvm *kvm, bool inject) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + vgic_v5_drain_pending_irqs(kvm, vmi, inject); + + return 0; +} + +/* Replay pending state that was cleared while restoring guest IST state. */ +int vgic_v5_restore_pending_irqs(struct kvm *kvm) +{ + return vgic_v5_process_pending_irqs(kvm, true); +} + +/* Drop pending state collected by a failed IST restore. */ +void vgic_v5_discard_pending_irqs(struct kvm *kvm) +{ + vgic_v5_process_pending_irqs(kvm, false); +}
diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h new file mode 100644 index 0000000..8909692 --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h
@@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#ifndef __KVM_ARM_VGICV5_TABLES_H__ +#define __KVM_ARM_VGICV5_TABLES_H__ + +#include <linux/spinlock.h> +#include <linux/idr.h> +#include <linux/irqchip/arm-gic-v5.h> +#include <linux/list.h> + +/* Level 1 Virtual Machine Table Entry */ +typedef __le64 vmtl1_entry; + +/* Level 2 Virtual Machine Table Entry */ +struct vmtl2_entry { + __le64 val[4]; +}; + +/* Virtual PE Table Entry */ +typedef __le64 vpe_entry; + +struct vgic_v5_vm_info { + void *vmd_base; + vpe_entry *vpet_base; + void *vped_base; + u8 vpe_id_bits; + + /* + * Both the LPI and SPI ISTs are allocated by the hypervisor. While it + * would be possible to track and access them by iterating over the ISTs + * themselves, it makes more sense to store pointers to the ISTs. + * + * The LPI IST can either be two-level or linear. Hence, we keep track + * of the structure. If it is two-level, we retain pointers to the L1 + * IST and to each L2 IST array. If it is linear, we just store the base + * address of the IST array. + * + * The SPI IST is linear, and therefore we just store the base address + * of the SPI IST array. + */ + bool h_lpi_ist_structure; + __le64 *h_lpi_ist; + __le64 **h_lpi_l2_ists; + __le64 *h_spi_ist; + + /* Tracking of pending interrupts as part of IST restore */ + struct list_head pending_irqs; +}; + +struct vgic_v5_vmt { + union { + struct { + struct vmtl2_entry *vmt_base; + unsigned int num_ents; + } linear; + struct { + vmtl1_entry *vmt_base; + struct vmtl2_entry **l2ptrs; + unsigned int num_l1_ents; + } l2; + }; + bool two_level; + unsigned int num_entries; + unsigned int max_vpes; + size_t vmd_size; + size_t vped_size; + struct ida vm_id_ida; +}; + +static inline u32 vgic_v5_vm_id(struct kvm *kvm) +{ + return kvm->arch.vgic.gicv5_vm.vm_id; +} + +/* + * For vGICv5, we need to consolidate two views: + * - The vcpu ID assigned by userspace, which it can assign however it + * sees fit. + * - The index into the VPET, which is also presented to the guest as + * the IAFFID for the VPE in question. + * + * Ideally, we don't want to intercept the guest setting an interrupt's affinity + * - this would somewhat defeat the point of allowing the hardware to manage the + * interrupt directly. Therefore, we need to use vcpu->vcpu_id as our VPE ID, + * which ensures that both userspace and the GICv5 hardware have a consistent + * view. + * + * This means two things: + * - Our VPET might be sparse, depending on the IDs picked by userspace. + * - We need to check the vcpu ID provided by userspace, and reject anything + * we cannot handle. + * + * In order to at least ensure consistent usage in the places that we need to + * use the VPE ID, we provide this helper. + */ +static inline u16 vgic_v5_vpe_id(struct kvm_vcpu *vcpu) +{ + return vcpu->vcpu_id; +} + +static inline int vgic_v5_vpe_db(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db; +} + +extern raw_spinlock_t vgic_v5_irs_lock; + +int vgic_v5_vmt_allocate(unsigned int max_vpes); +int vgic_v5_vmt_free(void); +phys_addr_t vgic_v5_get_vmt_base(void); + +int vgic_v5_allocate_vm_id(struct kvm *kvm); +void vgic_v5_release_vm_id(struct kvm *kvm); + +int vgic_v5_vmte_init(struct kvm *kvm); +int vgic_v5_vmte_release(struct kvm *kvm); +u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu); +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); + +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_exists(struct kvm *kvm); +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_free(struct kvm *kvm); + +int vgic_v5_save_spi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_save_lpi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_spi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_lpi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_pending_irqs(struct kvm *kvm); +void vgic_v5_discard_pending_irqs(struct kvm *kvm); + +#endif
diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 4e3e4db..91810fb 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c
@@ -7,10 +7,18 @@ #include <linux/bitops.h> #include <linux/irqchip/arm-vgic-info.h> +#include <linux/irqdomain.h> +#include <linux/kvm_host.h> +#include <linux/uaccess.h> +#include "vgic-v5-tables.h" #include "vgic.h" #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, phys_addr_t vmt_base); +static int vgic_v5_irs_clear_vmt(void); /* * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which @@ -34,6 +42,36 @@ static void vgic_v5_get_implemented_ppis(void) __assign_bit(GICV5_ARCH_PPI_PMUIRQ, ppi_caps.impl_ppi_mask, system_supports_pmuv3()); } +static u32 irs_readl_relaxed(const u32 reg_offset) +{ + return readl_relaxed(irs_caps.irs_base + reg_offset); +} + +static void irs_writel_relaxed(const u32 val, const u32 reg_offset) +{ + writel_relaxed(val, irs_caps.irs_base + reg_offset); +} + +static u64 irs_readq_relaxed(const u32 reg_offset) +{ + return readq_relaxed(irs_caps.irs_base + reg_offset); +} + +static void irs_writeq_relaxed(const u64 val, const u32 reg_offset) +{ + writeq_relaxed(val, irs_caps.irs_base + reg_offset); +} + +static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info) +{ + irs_caps.irs_base = info->gicv5_irs.base; + irs_caps.non_coherent = info->gicv5_irs.non_coherent; + + irs_caps.idr2 = irs_readl_relaxed(GICV5_IRS_IDR2); + irs_caps.idr3 = irs_readl_relaxed(GICV5_IRS_IDR3); + irs_caps.idr4 = irs_readl_relaxed(GICV5_IRS_IDR4); +} + /* * Probe for a vGICv5 compatible interrupt controller, returning 0 on success. */ @@ -43,6 +81,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info) int ret; kvm_vgic_global_state.type = VGIC_V5; + kvm_vgic_global_state.max_gic_vcpus = 0; + kvm_vgic_global_state.max_gicv5_vcpus = 0; kvm_vgic_global_state.vcpu_base = 0; kvm_vgic_global_state.vctrl_base = NULL; @@ -60,17 +100,65 @@ int vgic_v5_probe(const struct gic_kvm_info *info) goto skip_v5; } - kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS; + if (!gicv5_global_data.lpi_domain) { + kvm_err("GICv5 LPI domain unavailable\n"); + return -ENODEV; + } + vgic_v5_irs_cache_id_regs(info); vgic_v5_get_implemented_ppis(); + /* + * Even if the HW supports more per-VM vCPUs, artificially cap as we + * can't use them all. + */ + kvm_vgic_global_state.max_gicv5_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), + VGIC_V5_MAX_CPUS); + + /* + * GICv5 requires a set of tables to be allocated in order to manage + * VMs. We allocate them in advance here, which alas means that we + * already have to make a decisions regarding the maximum number of VMs + * we want to run. For now, we match the maximum number offered by the + * hardware, but this might not be a wise choice in the long term. + */ + ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gicv5_vcpus); + if (ret) { + kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n"); + return -ENODEV; + } + + /* + * We've now allocated the VM table, but the host's IRS doesn't know + * about it yet. Provide the base address of the VMT to the IRS, as well + * as the number of ID bits that it covers and the structure used + * (linear/two-level). + */ + ret = vgic_v5_irs_assign_vmt(vgic_v5_irs_two_level_vmt_support(&irs_caps), + ilog2(vgic_v5_irs_max_vms(&irs_caps)), + vgic_v5_get_vmt_base()); + if (ret) { + kvm_err("Failed to assign the GICv5 VM tables to the IRS; no GICv5 support\n"); + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; + } + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5); if (ret) { kvm_err("Cannot register GICv5 KVM device.\n"); - goto skip_v5; + /* + * Don't free the VMT itself if the hardware still has a valid + * pointer to it. + */ + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; } v5_registered = true; + kvm_vgic_global_state.max_gic_vcpus = + kvm_vgic_global_state.max_gicv5_vcpus; kvm_info("GCIE system register CPU interface\n"); skip_v5: @@ -92,12 +180,12 @@ int vgic_v5_probe(const struct gic_kvm_info *info) ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { kvm_err("Cannot register GICv3-legacy KVM device.\n"); - return ret; + /* vGICv5 should still work */ + return v5_registered ? 0 : ret; } - /* We potentially limit the max VCPUs further than we need to here */ - kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS, - VGIC_V5_MAX_CPUS); + kvm_vgic_global_state.max_gic_vcpus = max(kvm_vgic_global_state.max_gic_vcpus, + VGIC_V3_MAX_CPUS); static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); kvm_info("GCIE legacy system register CPU interface\n"); @@ -107,6 +195,411 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } +/* + * Wait for completion of a change in any of IRS_VMT_BASER, IRS_VMAP_L2_VMTR, + * IRS_VMAP_VMR, IRS_VMAP_VPER, IRS_VMAP_VISTR, IRS_VMAP_L2_VISTR. + */ +static int vgic_v5_irs_wait_for_vm_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VMT_STATUSR, + GICV5_IRS_VMT_STATUSR_IDLE, + NULL); +} + +/* + * Wait for completion of a change in any of IRS_VPE_SELR, IRS_VPE_DBR, + * IRS_VPE_CR0. + */ +static int vgic_v5_irs_wait_for_vpe_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VPE_STATUSR, + GICV5_IRS_VPE_STATUSR_IDLE, + NULL); +} + +/* + * Wait for a write to IRS_SAVE_VMR to complete. + */ +static int vgic_v5_irs_wait_for_save_vm_op(u32 *statusr) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_SAVE_VM_STATUSR, + GICV5_IRS_SAVE_VM_STATUSR_IDLE, + statusr); +} + +static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) +{ + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + irs_writeq_relaxed(val, offset); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, + phys_addr_t vmt_base) +{ + u64 vmt_baser; + u32 vmt_cfgr; + int ret; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + vmt_baser = irs_readq_relaxed(GICV5_IRS_VMT_BASER); + if (!!FIELD_GET(GICV5_IRS_VMT_BASER_VALID, vmt_baser)) + return -EBUSY; + + vmt_cfgr = FIELD_PREP(GICV5_IRS_VMT_CFGR_VM_ID_BITS, vm_id_bits); + if (two_level) + vmt_cfgr |= FIELD_PREP(GICV5_IRS_VMT_CFGR_STRUCTURE, + GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL); + + irs_writel_relaxed(vmt_cfgr, GICV5_IRS_VMT_CFGR); + + /* The base address is intentionally only masked and not shifted */ + vmt_baser = FIELD_PREP(GICV5_IRS_VMT_BASER_VALID, true) | + (vmt_base & GICV5_IRS_VMT_BASER_ADDR); + irs_writeq_relaxed(vmt_baser, GICV5_IRS_VMT_BASER); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_clear_vmt(void) +{ + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + return vgic_v5_irs_write_vm_mmio_reg(0, GICV5_IRS_VMT_BASER); +} + +static int vgic_v5_irs_vmap_l2_vmt(u16 vm_id) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_L2_VMTR_VM_ID, vm_id) | + GICV5_IRS_VMAP_L2_VMTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_L2_VMTR); +} + +static int __vgic_v5_irs_vmap_vm(u16 vm_id, bool unmap) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VMR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VMR_U, unmap) | + GICV5_IRS_VMAP_VMR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VMR); +} + +static int vgic_v5_irs_set_vm_valid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, false); +} + +static int vgic_v5_irs_set_vm_invalid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, true); +} + +static int __vgic_v5_irs_update_vist_validity(u16 vm_id, bool spi_ist, bool unmap) +{ + u8 type = spi_ist ? 0b011 : 0b010; + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VISTR_TYPE, type) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_U, unmap) | + GICV5_IRS_VMAP_VISTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VISTR); +} + +static int vgic_v5_irs_set_vist_valid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, false); +} + +/* + * LPI ISTs can be invalidated explicitly. SPI ISTs are invalidated by making + * the VMTE invalid during teardown. + */ +static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true); +} + +static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id, + irq_hw_number_t db_hwirq) +{ + u64 vmap_vper, dbr, selr; + u32 statusr, cr0; + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + /* Mark the VPE as valid */ + vmap_vper = FIELD_PREP(GICV5_IRS_VMAP_VPER_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VMAP_VPER_VM_ID, vm_id) | + GICV5_IRS_VMAP_VPER_M; + irs_writeq_relaxed(vmap_vper, GICV5_IRS_VMAP_VPER); + + /* Wait for the VPE to be marked valid in the VPET */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + selr = FIELD_PREP(GICV5_IRS_VPE_SELR_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VPE_SELR_VM_ID, vm_id) | + GICV5_IRS_VPE_SELR_S; + irs_writeq_relaxed(selr, GICV5_IRS_VPE_SELR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + statusr = irs_readl_relaxed(GICV5_IRS_VPE_STATUSR); + if (!FIELD_GET(GICV5_IRS_VPE_STATUSR_V, statusr)) + return -EINVAL; + + /* Set targeted only routing (disable 1ofN vPE selection) */ + cr0 = GICV5_IRS_VPE_CR0_DPS; + irs_writel_relaxed(cr0, GICV5_IRS_VPE_CR0); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + /* + * The VPE has not yet run. Therefore, make sure that all interrupts + * will generate a doorbell. + */ + dbr = FIELD_PREP(GICV5_IRS_VPE_DBR_INTID, db_hwirq) | + GICV5_IRS_VPE_DBR_DBV; + irs_writeq_relaxed(dbr, GICV5_IRS_VPE_DBR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + return 0; +} + +static int vgic_v5_irs_save_vm_op(u16 vm_id, bool save, u32 *statusr) +{ + u64 save_vmr; + int ret; + + save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id); + save_vmr |= GICV5_IRS_SAVE_VMR_Q; + save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, save); + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with. */ + ret = vgic_v5_irs_wait_for_save_vm_op(NULL); + if (ret) + return ret; + + irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR); + + return vgic_v5_irs_wait_for_save_vm_op(statusr); +} + +static irqreturn_t db_handler(int irq, void *data) +{ + struct kvm_vcpu *vcpu = data; + + WRITE_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired, true); + + kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); + kvm_vcpu_kick(vcpu); + + return IRQ_HANDLED; +} + +static int vgic_v5_send_command(struct kvm_vcpu *vcpu, enum gicv5_vcpu_cmd cmd) +{ + int irq = vgic_v5_vpe_db(vcpu); + + if (!irq) + return -ENXIO; + + return irq_set_vcpu_affinity(irq, &cmd); +} + +static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) +{ + struct vgic_v5_vm *vm = data->domain->host_data; + enum gicv5_vcpu_cmd *cmd = vcpu_info; + unsigned int vcpu_idx = data->hwirq; + struct kvm_vcpu *vcpu; + u16 vpe_id; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + switch (*cmd) { + case VMT_L2_MAP: + return vgic_v5_irs_vmap_l2_vmt(vm->vm_id); + case VMTE_MAKE_VALID: + return vgic_v5_irs_set_vm_valid(vm->vm_id); + case VMTE_MAKE_INVALID: + return vgic_v5_irs_set_vm_invalid(vm->vm_id); + case VPE_MAKE_VALID: + /* + * The index in the doorbell domain aligns with our flat + * vcpu_idx index. However, we need the actual VPE ID, which + * means we first need to resolve the actual vcpu. + */ + vcpu = kvm_get_vcpu(vm->kvm, vcpu_idx); + if (!vcpu) + return -EINVAL; + + vpe_id = vgic_v5_vpe_id(vcpu); + + /* + * We need the actual LPI ID which lives in the top-most parent + * domain. This hwirq won't include the type (LPI) but that's + * not required for the IRS_VPE_DBR. + */ + while (data->parent_data) + data = data->parent_data; + return vgic_v5_irs_set_up_vpe(vm->vm_id, vpe_id, data->hwirq); + case SPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, true); + case LPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, false); + case LPI_VIST_MAKE_INVALID: + return vgic_v5_irs_set_vist_invalid(vm->vm_id, false); + default: + return -EINVAL; + } +} + +/* + * This set of irq_chip functions is specific for doorbells. + */ +static const struct irq_chip vgic_v5_db_irq_chip = { + .name = "GICv5-DB", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_set_vcpu_affinity = vgic_v5_db_set_vcpu_affinity, + .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static void vgic_v5_irq_db_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + const struct irq_chip *chip = &vgic_v5_db_irq_chip; + struct irq_data *irqd; + int ret; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL); + if (ret) + return ret; + + for (int i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(domain, virq + i, i, chip, + domain->host_data); + irqd = irq_desc_get_irq_data(irq_to_desc(virq + i)); + irqd_set_single_target(irqd); + } + + return 0; +} + +static const struct irq_domain_ops vgic_v5_irq_db_domain_ops = { + .alloc = vgic_v5_irq_db_domain_alloc, + .free = vgic_v5_irq_db_domain_free, +}; + +static int vgic_v5_create_per_vm_domain(struct kvm *kvm) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + int nr_vcpus = atomic_read(&kvm->online_vcpus); + int id = task_pid_nr(current); + int ret, db_virq = 0; + + vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id); + if (!vm->fwnode) + return -ENOMEM; + + vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, + 0, nr_vcpus, vm->fwnode, + &vgic_v5_irq_db_domain_ops, vm); + if (!vm->domain) { + ret = -ENOMEM; + goto err; + } + + db_virq = irq_domain_alloc_irqs(vm->domain, nr_vcpus, NUMA_NO_NODE, vm); + if (db_virq <= 0) { + ret = db_virq; + goto err; + } + + kvm->arch.vgic.gicv5_vm.vpe_db_base = db_virq; + + return 0; + +err: + if (vm->domain) + irq_domain_remove(vm->domain); + if (vm->fwnode) + irq_domain_free_fwnode(vm->fwnode); + + kvm->arch.vgic.gicv5_vm.vpe_db_base = 0; + vm->domain = NULL; + vm->fwnode = NULL; + + return ret; +} + +static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm) +{ + if (!vm->domain) + return; + + irq_domain_remove(vm->domain); + irq_domain_free_fwnode(vm->fwnode); + vm->domain = NULL; + vm->fwnode = NULL; +} + void vgic_v5_reset(struct kvm_vcpu *vcpu) { /* @@ -120,17 +613,117 @@ void vgic_v5_reset(struct kvm_vcpu *vcpu) * CPUIF (but potentially fewer in the IRS). */ vcpu->arch.vgic_cpu.num_pri_bits = 5; + + /* Make the VPE valid in the VPET */ + if (WARN_ON(vgic_v5_send_command(vcpu, VPE_MAKE_VALID))) + return; } +static void vgic_v5_free_doorbells(struct kvm *kvm, unsigned int nr_dbs) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + struct kvm_vcpu *vcpu; + unsigned long i; + int db; + + for (i = 0; i < nr_dbs; i++) { + vcpu = kvm_get_vcpu(kvm, i); + db = vgic_v5_vpe_db(vcpu); + if (!db) + continue; + + free_irq(db, vcpu); + vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = 0; + } + + if (vm->vpe_db_base) { + irq_domain_free_irqs(vm->vpe_db_base, + atomic_read(&kvm->online_vcpus)); + vm->vpe_db_base = 0; + } +} + +void vgic_v5_teardown(struct kvm *kvm) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu, *vcpu0; + bool release_vm_id = true; + unsigned long i; + int rc; + + lockdep_assert_held(&kvm->arch.config_lock); + + /* + * If the VM's ID isn't valid, then we either failed init very early or + * we've been called a second time. Nothing to do here in either case. + */ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + if (kvm->arch.vgic.gicv5_vm.vmte_allocated) { + /* Make the VM invalid */ + vcpu0 = kvm_get_vcpu(kvm, 0); + rc = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID); + if (rc) { + kvm_err("could not make VMTE invalid\n"); + release_vm_id = false; + goto out_free_doorbells; + } + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vgic_v5_vmte_free_vpe(vcpu)) { + kvm_err("Failed to free VPE\n"); + release_vm_id = false; + goto out_free_doorbells; + } + } + + if (vgic_v5_vmte_release(kvm)) { + kvm_err("Failed to release VM 0x%x\n", dist->gicv5_vm.vm_id); + release_vm_id = false; + } + } + +out_free_doorbells: + vgic_v5_free_doorbells(kvm, atomic_read(&kvm->online_vcpus)); + vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm); + + /* + * We only release the VM ID itself if we didn't fail earlier. It does + * mean that we might lose the VM ID (and associated VMTE, etc), but + * given that we've failed to tear them down correctly there's no way to + * safely reuse them. The VM ID allocating IDA will make sure we don't + * accidentally reuse this partially torn down state. + */ + if (release_vm_id) + vgic_v5_release_vm_id(kvm); +} + +/* + * Claim and populate a VMTE (optionally making a new L2 VMT valid), create VPE + * doorbells, allocate VPET and populate for each VPE. Finally, we also init the + * vIRS, which means allocating and making the virtual SPI IST valid. + * + * Note: We do need to put the cart before the horse here. The VPE doorbells are + * our conduit for communication with the IRS, which means we need to have those + * before making the VMTE valid. + * + * On failure, we clean up in the teardown path (vgic_v5_teardown()). + */ int vgic_v5_init(struct kvm *kvm) { - struct kvm_vcpu *vcpu; - unsigned long idx; + struct kvm_vcpu *vcpu, *vcpu0; + int nr_vcpus, ret = 0; + unsigned int db_virq; + unsigned long i; - if (vgic_initialized(kvm)) - return 0; + lockdep_assert_held(&kvm->arch.config_lock); - kvm_for_each_vcpu(idx, vcpu, kvm) { + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus == 0) + return -ENODEV; + + kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu_has_nv(vcpu)) { kvm_err("Nested GICv5 VMs are currently unsupported\n"); return -EINVAL; @@ -145,20 +738,84 @@ int vgic_v5_init(struct kvm *kvm) kvm->arch.vgic.gicv5_vm.userspace_ppis, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + ret = vgic_v5_allocate_vm_id(kvm); + if (ret) + return ret; + + /* + * Stash a backpointer to struct kvm. It is required to resolve the VPE + * ID from the doorbell index, which matches the flat vcpu_idx and not + * the vcpu_id. + */ + kvm->arch.vgic.gicv5_vm.kvm = kvm; + + ret = vgic_v5_create_per_vm_domain(kvm); + if (ret) + goto err; + + db_virq = kvm->arch.vgic.gicv5_vm.vpe_db_base; + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = request_irq(db_virq + i, db_handler, 0, "vcpu", vcpu); + if (ret) + goto err; + + /* Stash it with the VCPU for easy retrieval */ + vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = db_virq + i; + } + + /* Populate VMTE (with VPET and VM descriptor) */ + ret = vgic_v5_vmte_init(kvm); + if (ret) + goto err; + + /* We pick the first vcpu to make the VMTE valid - any would do */ + vcpu0 = kvm_get_vcpu(kvm, 0); + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID); + if (ret) + goto err; + + /* Populate the VPETE for each VPE. */ + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = vgic_v5_vmte_alloc_vpe(vcpu); + if (ret) + goto err; + } + + /* Init IRS (and alloc SPI IST) */ + ret = kvm_vgic_v5_irs_init(kvm, kvm->arch.vgic.nr_spis); + if (ret) + goto err; + return 0; + +err: + /* + * Explicitly tear everything down on failure. The teardown function is + * written to handle any partial state we might have, so we don't need + * to do any clean-up first. Teardown will be called a second time on VM + * destruction, but that's fine - it is better to leave things in a + * clean state now, and doubly so because userspace could actually go + * and retry init. + */ + vgic_v5_teardown(kvm); + + return ret; } int vgic_v5_map_resources(struct kvm *kvm) { if (!vgic_initialized(kvm)) return -EBUSY; + if (kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending) + return -EINVAL; return 0; } -int vgic_v5_finalize_ppi_state(struct kvm *kvm) +int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu) { - struct kvm_vcpu *vcpu0; + struct kvm *kvm = vcpu->kvm; + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; int i; if (!vgic_is_v5(kvm)) @@ -167,35 +824,65 @@ int vgic_v5_finalize_ppi_state(struct kvm *kvm) guard(mutex)(&kvm->arch.config_lock); /* - * If SW_PPI has been advertised, then we know we already - * initialised the whole thing, and we can return early. Yes, - * this is pretty hackish as far as state tracking goes... + * Discover the set of PPIs that are exposed to the guest once per VM. + * Once known, apply that mask to each VCPU's restored PPI state as the + * VCPUs are first run. */ - if (test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) - return 0; + if (!test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) { + bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, + VGIC_V5_NR_PRIVATE_IRQS); + bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, + VGIC_V5_NR_PRIVATE_IRQS); - /* The PPI state for all VCPUs should be the same. Pick the first. */ - vcpu0 = kvm_get_vcpu(kvm, 0); + for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) { + const u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; - bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); - bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + irq = vgic_get_vcpu_irq(vcpu, intid); - for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) { + /* Expose PPIs with an owner or the SW_PPI, only */ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) { + __set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); + __assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, + irq->config == VGIC_CONFIG_LEVEL); + } + } + + vgic_put_irq(kvm, irq); + } + } + + /* + * Apply the mask to Enable, Active. Skip pending as that's calculated + * on guest entry. + */ + bitmap_and(cpu_if->vgic_ppi_enabler, cpu_if->vgic_ppi_enabler, + kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + bitmap_and(cpu_if->vgic_ppi_activer, cpu_if->vgic_ppi_activer, + kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + + /* Also update the vgic_irqs */ + for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) { + bool visible = test_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); const u32 intid = vgic_v5_make_ppi(i); struct vgic_irq *irq; - irq = vgic_get_vcpu_irq(vcpu0, intid); + irq = vgic_get_vcpu_irq(vcpu, intid); - /* Expose PPIs with an owner or the SW_PPI, only */ scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { - if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) { - __set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); - __assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, - irq->config == VGIC_CONFIG_LEVEL); + if (!visible) { + irq->enabled = false; + irq->active = false; + irq->pending_latch = false; + irq->line_level = false; + } else { + irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler); + irq->active = test_bit(i, cpu_if->vgic_ppi_activer); } } - vgic_put_irq(vcpu0->kvm, irq); + vgic_put_irq(kvm, irq); } return 0; @@ -436,9 +1123,53 @@ void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu) VGIC_V5_NR_PRIVATE_IRQS); } +static bool vgic_v5_set_spi_pending_state(struct kvm_vcpu *vcpu, + struct vgic_irq *irq) +{ + vgic_v5_set_irq_pend(irq->target_vcpu, irq); + return true; +} + +bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, + struct vgic_irq *irq, + unsigned long flags) + __releases(&irq->irq_lock) +{ + lockdep_assert_held(&irq->irq_lock); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + return true; +} + +static const struct irq_ops vgic_v5_spi_irq_ops = { + .set_pending_state = vgic_v5_set_spi_pending_state, + .queue_irq_unlock = vgic_v5_spi_queue_irq_unlock, +}; + +void vgic_v5_set_spi_ops(struct vgic_irq *irq) +{ + if (WARN_ON(!irq) || WARN_ON(irq->ops)) + return; + + irq->ops = &vgic_v5_spi_irq_ops; +} + +/* Set the pending state for GICv5 SPIs and LPIs */ +void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq) +{ + if (WARN_ON(__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid))) + return; + + kvm_call_hyp(__vgic_v5_vdpend, irq->intid, irq_is_pending(irq), + vcpu->kvm->arch.vgic.gicv5_vm.vm_id); +} + void vgic_v5_load(struct kvm_vcpu *vcpu) { + bool irichppidis = !READ_ONCE(vcpu->kvm->arch.vgic.enabled); struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + u16 vm = vgic_v5_vm_id(vcpu->kvm); + u16 vpe = vgic_v5_vpe_id(vcpu); /* * On the WFI path, vgic_load is called a second time. The first is when @@ -451,7 +1182,16 @@ void vgic_v5_load(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if); - cpu_if->gicv5_vpe.resident = true; + cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_V, true) | + FIELD_PREP(ICH_CONTEXTR_EL2_IRICHPPIDIS, irichppidis) | + FIELD_PREP(ICH_CONTEXTR_EL2_VPE, vpe) | + FIELD_PREP(ICH_CONTEXTR_EL2_VM, vm); + + kvm_call_hyp(__vgic_v5_make_resident, cpu_if); + + /* Failed to make the VPE resident? Bang! */ + if (WARN_ON(!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr))) + kvm_vm_dead(vcpu->kvm); } void vgic_v5_put(struct kvm_vcpu *vcpu) @@ -469,7 +1209,58 @@ void vgic_v5_put(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_save_apr, cpu_if); - cpu_if->gicv5_vpe.resident = false; + cpu_if->vgic_contextr = 0; + + /* Request a doorbell if entering WFI, unless the IRS is disabled */ + if (vcpu_get_flag(vcpu, IN_WFI) && + READ_ONCE(vcpu->kvm->arch.vgic.enabled)) { + u32 priority_mask; + int dbpm; + + /* + * Find the virtual running priority and use this to calculate + * the doorbell priority mask. We combine the highest active + * priority and the CPU's priority mask. The guest can't handle + * interrupts with priorities less than or equal to the virtual + * running priority, so there's literally no point in waking the + * guest for these. + * + * The priority needs to be higher than the mask to signal, so + * pick the next higher priority (subtract 1). + */ + priority_mask = vgic_v5_get_effective_priority_mask(vcpu); + + /* + * Request a doorbell *unless* the priority is 0, indicating + * that no interrupt can wake the CPU up. + */ + if (priority_mask) { + int db_irq = vgic_v5_vpe_db(vcpu); + struct irq_data *d = irq_get_irq_data(db_irq); + const struct cpumask *aff = irq_data_get_effective_affinity_mask(d); + int cpu = smp_processor_id(); + + dbpm = priority_mask - 1; + cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_DB, 1) | + FIELD_PREP(ICH_CONTEXTR_EL2_DBPM, dbpm); + + /* + * Make the doorbell affine to this CPU, if it isn't + * already. Actively check the cpumask first as it is + * cheaper than changing the affinity every time. + */ + if (!cpumask_test_cpu(cpu, aff)) { + int ret; + + ret = irq_set_affinity(db_irq, cpumask_of(cpu)); + if (ret) + kvm_debug_ratelimited("Failed to move doorbell IRQ %d to CPU %d: %d\n", + db_irq, cpu, ret); + } + } + } + + kvm_call_hyp(__vgic_v5_make_non_resident, cpu_if); /* The shadow priority is only updated on entering WFI */ if (vcpu_get_flag(vcpu, IN_WFI)) @@ -513,3 +1304,300 @@ void vgic_v5_save_state(struct kvm_vcpu *vcpu) __vgic_v5_save_ppi_state(cpu_if); dsb(sy); } + +static int vgic_v5_irs_status_is_quiesced(u32 statusr) +{ + if (statusr & GICV5_IRS_SAVE_VM_STATUSR_Q) + return 0; + + return -EBUSY; +} + +static int vgic_v5_irs_is_quiesced(u16 vm_id) +{ + u32 statusr; + int ret; + + /* + * Ensure that the IST snapshot is complete before asking the IRS + * whether the VM remained quiescent while it was copied. + */ + mb(); + + ret = vgic_v5_irs_save_vm_op(vm_id, false, &statusr); + if (ret) + return ret; + + return vgic_v5_irs_status_is_quiesced(statusr); +} + +static int vgic_v5_copy_ist_attr(struct kvm_device_attr *attr, + struct kvm_vgic_v5_ist *ist_attr) +{ + void __user *uaddr = (void __user *)(unsigned long)attr->addr; + + if (!uaddr) + return -EINVAL; + + if (copy_from_user(ist_attr, uaddr, sizeof(*ist_attr))) + return -EFAULT; + + return 0; +} + +static int vgic_v5_validate_ist_user_buffer(__u64 addr, __u64 size, + size_t expected) +{ + if (!addr || size != expected) + return -EINVAL; + + return 0; +} + +static int vgic_v5_validate_ist_attr(struct kvm *kvm, + const struct kvm_vgic_v5_ist *ist_attr) +{ + unsigned int id_bits; + int ret; + + /* We always have SPIs to save */ + ret = vgic_v5_validate_ist_user_buffer(ist_attr->spi_ist_addr, + ist_attr->spi_ist_size, + kvm->arch.vgic.nr_spis * sizeof(__u32)); + if (ret) + return ret; + + /* We don't always have LPIs to save */ + ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits); + if (ret < 0) + return ret; + + /* No LPI IST */ + if (!ret) { + if (ist_attr->lpi_ist_addr || ist_attr->lpi_ist_size) + return -EINVAL; + + return 0; + } + + return vgic_v5_validate_ist_user_buffer(ist_attr->lpi_ist_addr, + ist_attr->lpi_ist_size, + BIT(id_bits) * sizeof(__u32)); +} + +int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr) +{ + struct kvm_vgic_v5_ist ist_attr; + u16 vm_id = vgic_v5_vm_id(kvm); + u32 statusr; + int ret = 0; + + mutex_lock(&kvm->lock); + + if (kvm_trylock_all_vcpus(kvm)) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + mutex_lock(&kvm->arch.config_lock); + + if (!vgic_initialized(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = vgic_v5_copy_ist_attr(attr, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_validate_ist_attr(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_save_vm_op(vm_id, true, &statusr); + if (ret) { + kvm_err("Failed to save GICv5 IRS VM state: %d\n", ret); + goto out_unlock; + } + + ret = vgic_v5_irs_status_is_quiesced(statusr); + if (ret) + goto out_unlock; + + /* Save the SPI IST to the userspace buffer. */ + ret = vgic_v5_save_spi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_is_quiesced(vm_id); + if (ret) + goto out_unlock; + + /* Save the LPI IST to the userspace buffer. */ + ret = vgic_v5_save_lpi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_is_quiesced(vm_id); + if (ret) + goto out_unlock; + +out_unlock: + mutex_unlock(&kvm->arch.config_lock); + kvm_unlock_all_vcpus(kvm); + mutex_unlock(&kvm->lock); + + return ret; +} + +/* Allocate the LPI IST to restore into */ +static int vgic_v5_restore_lpi_ist_alloc(struct kvm *kvm, bool *allocated) +{ + unsigned int id_bits; + int ret; + + *allocated = false; + + ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits); + if (ret <= 0) + return ret; + + ret = vgic_v5_lpi_ist_alloc(kvm, id_bits); + if (ret) + return ret; + + *allocated = true; + + return 0; +} + +/* + * Clean up the LPI IST if we allocated it, and restore the VMTE to the + * original, valid state. + */ +static void vgic_v5_restore_cleanup(struct kvm *kvm, + struct kvm_vcpu *vcpu, + bool lpi_ist_allocated) +{ + /* + * We are on the restore failure path, so we do a best-effort + * cleanup. These commands might fail, but at this stage this is the + * best we can realistically do. + */ + if (lpi_ist_allocated) { + if (!vgic_v5_send_command(vcpu, VMTE_MAKE_INVALID)) + vgic_v5_lpi_ist_free(kvm); + } + + vgic_v5_send_command(vcpu, VMTE_MAKE_VALID); +} + +int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr) +{ + bool lpi_ist_allocated = false, vmte_invalid = false; + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + struct kvm_vgic_v5_ist ist_attr; + int ret = 0; + + mutex_lock(&kvm->lock); + + if (kvm_trylock_all_vcpus(kvm)) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + mutex_lock(&kvm->arch.config_lock); + + if (!vgic_initialized(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + if (kvm_vm_has_ran_once(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = vgic_v5_copy_ist_attr(attr, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_validate_ist_attr(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_lpi_ist_exists(kvm); + if (ret) { + if (ret > 0) + ret = -EBUSY; + goto out_unlock; + } + + /* + * If the guest has previously allocated an IST (which we check based on + * the IRS_IST_BASER), extract the number of LPI ID bits from the + * IRS_IST_CFGR. Else, do nothing. + * + * We do this before making the VMTE invalid as we rely on + * IRS_VMAP_VISTR to mark the IST as valid in the VMTE. This can only + * happen while the VMTE is valid. + */ + ret = vgic_v5_restore_lpi_ist_alloc(kvm, &lpi_ist_allocated); + if (ret) + goto out_unlock; + + /* + * Host ISTs are updated while the VMTE is invalid, so the GIC cannot + * observe partially restored state. + */ + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID); + if (ret) { + /* + * If invalidation fails, the restore cannot safely update host + * IST state. + */ + goto out_unlock; + } + vmte_invalid = true; + + /* Restore the SPI IST from the userspace buffer. */ + ret = vgic_v5_restore_spi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + /* Restore the LPI IST from the userspace buffer. */ + if (lpi_ist_allocated) { + ret = vgic_v5_restore_lpi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + } + + /* And make the VM Valid again */ + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID); + if (ret) + goto out_unlock; + vmte_invalid = false; + + /* + * As part of restoring the ISTs, and previously pending interrupts have + * been tracked and made non-pending. Now that the ISTs have been + * restored, and the VM is valid again, restore the pending interrupts. + */ + ret = vgic_v5_restore_pending_irqs(kvm); + if (ret) + goto out_unlock; + + kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending = false; + +out_unlock: + if (ret && (vmte_invalid || lpi_ist_allocated)) { + vgic_v5_discard_pending_irqs(kvm); + vgic_v5_restore_cleanup(kvm, vcpu0, lpi_ist_allocated); + } + + mutex_unlock(&kvm->arch.config_lock); + kvm_unlock_all_vcpus(kvm); + mutex_unlock(&kvm->lock); + + return ret; +}
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5cf5a1e..c2726d4 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c
@@ -86,20 +86,32 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) */ struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid) { - /* Non-private IRQs are not yet implemented for GICv5 */ - if (vgic_is_v5(kvm)) - return NULL; + enum kvm_device_type type = kvm->arch.vgic.vgic_model; /* SPIs */ - if (intid >= VGIC_NR_PRIVATE_IRQS && - intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) { - intid -= VGIC_NR_PRIVATE_IRQS; - intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); - return &kvm->arch.vgic.spis[intid]; + if (__irq_is_spi(type, intid)) { + switch (type) { + case KVM_DEV_TYPE_ARM_VGIC_V5: + intid = vgic_v5_get_hwirq_id(intid); + + if (intid >= kvm->arch.vgic.nr_spis) + return NULL; + + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; + default: { + u32 max_intid = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; + + if (intid < max_intid) { + intid -= VGIC_NR_PRIVATE_IRQS; + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; + } + }} } /* LPIs */ - if (irq_is_lpi(kvm, intid)) + if (__irq_is_lpi(type, intid)) return vgic_get_lpi(kvm, intid); return NULL; @@ -565,6 +577,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, else irq->pending_latch = true; + if (irq->ops && irq->ops->set_pending_state) + WARN_ON_ONCE(!irq->ops->set_pending_state(vcpu, irq)); + vgic_queue_irq_unlock(kvm, irq, flags); vgic_put_irq(kvm, irq); @@ -1247,8 +1262,12 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) unsigned long flags; struct vgic_vmcr vmcr; - if (vgic_is_v5(vcpu->kvm)) + if (vgic_is_v5(vcpu->kvm)) { + if (READ_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired)) + return true; + return vgic_v5_has_pending_ppi(vcpu); + } if (!vcpu->kvm->arch.vgic.enabled) return false;
diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b71d486..97fc3bdf 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h
@@ -269,6 +269,8 @@ struct ap_list_summary { #define irqs_active_outside_lrs(s) \ ((s)->nr_act && irqs_outside_lrs(s)) +int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr); int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, struct vgic_reg_attr *reg_attr); int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, @@ -383,8 +385,14 @@ void vgic_debug_destroy(struct kvm *kvm); int vgic_v5_probe(const struct gic_kvm_info *info); void vgic_v5_reset(struct kvm_vcpu *vcpu); int vgic_v5_init(struct kvm *kvm); +int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis); +void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); +void vgic_v5_set_spi_ops(struct vgic_irq *irq); +bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, + unsigned long flags); +void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq); bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu); void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu); void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu); @@ -394,6 +402,19 @@ void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_restore_state(struct kvm_vcpu *vcpu); void vgic_v5_save_state(struct kvm_vcpu *vcpu); +int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address); +int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits); + +int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr, bool is_write); +int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); +const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz); +int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr); +int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr); +int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + u64 *reg, bool is_write); +int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr); #define for_each_visible_v5_ppi(__i, __k) \ for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS)
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 2a8de20..4e7682df 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c
@@ -349,7 +349,7 @@ void __init arch_mm_preinit(void) unsigned long size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); - swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); + swiotlb_adjust_size(min(swiotlb_default_pool_size(), size)); } swiotlb_init(true, flags);
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 8108012..aaa34c7 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c
@@ -11,6 +11,7 @@ #include <linux/bitfield.h> #include <linux/bpf.h> #include <linux/cfi.h> +#include <linux/extable.h> #include <linux/filter.h> #include <linux/memory.h> #include <linux/printk.h> @@ -22,7 +23,6 @@ #include <asm/debug-monitors.h> #include <asm/insn.h> #include <asm/text-patching.h> -#include <asm/set_memory.h> #include "bpf_jit.h" @@ -1045,8 +1045,9 @@ static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx) } } -static void build_epilogue(struct jit_ctx *ctx, bool was_classic) +static void build_epilogue(struct jit_ctx *ctx, bool was_classic, bool unwind) { + const u8 tmp = bpf2a64[TMP_REG_1]; const u8 r0 = bpf2a64[BPF_REG_0]; const u8 ptr = bpf2a64[TCCNT_PTR]; @@ -1074,6 +1075,10 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic) if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) emit(A64_AUTIASP, ctx); + /* tmp is the offset of the pad, see emit_unwind_exit() */ + if (unwind) + emit(A64_ADD(1, A64_LR, A64_LR, tmp), ctx); + emit(A64_RET(A64_LR), ctx); } @@ -1222,6 +1227,82 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +/* + * There is no branch for unwind_goto, only an extable entry + * { .insn = ret_addr - 1, .fixup = pad - ret_addr } + * where ret_addr is the return address of the call in front of it. unwind_exit + * adds .fixup to its LR. + * ret_addr - 1, because what follows the call can have an entry of its own. + * No insn starts there, so a fault cannot hit the entry. + * 'bl foo; mov x8, x0' is in front of unwind_goto. The pad doesn't miss the mov: + * r0 is x8 and the epilogue of the callee copies it to x0 and leaves x8 as it is. + */ +static int add_unwind_exentry(struct jit_ctx *ctx, int pad) +{ + int ret_addr = ctx->idx - 1; + struct exception_table_entry *ex; + off_t ins_offset; + + if (!ctx->image) + /* First pass */ + return 0; + + if (!ctx->prog->aux->extable || + WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) + return -EINVAL; + + ex = &ctx->prog->aux->extable[ctx->exentry_idx]; + ins_offset = (long)&ctx->ro_image[ret_addr] - 1 - (long)&ex->insn; + if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) + return -ERANGE; + + /* switch ex to rw buffer for writing */ + ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); + ex->insn = ins_offset; + ex->fixup = (ctx->offset[pad] - ret_addr) * AARCH64_INSN_SIZE; + ex->type = EX_TYPE_BPF; + + ctx->exentry_idx++; + return 0; +} + +static long bpf_unwind_fixup(u64 lr) +{ + const struct exception_table_entry *e; + + e = search_bpf_extables(ptrauth_strip_kernel_insn_pac(lr) - 1); + /* + * The verifier made sure that unwind_goto follows a call in a prog. When + * the kernel called an async callback, it's exit. + */ + return e ? e->fixup : 0; +} + +/* + * unwind_exit: return to the pad. LR in the frame record is signed. Instead of + * signing the address of the pad, let the epilogue authenticate LR as it is and + * add the offset of the pad to it. r0 and r2 are returned. + */ +static void emit_unwind_exit(struct jit_ctx *ctx) +{ + const u8 tmp = bpf2a64[TMP_REG_1]; + const u8 r0 = bpf2a64[BPF_REG_0]; + const u8 r2 = bpf2a64[BPF_REG_2]; + + emit(A64_PUSH(r0, r2, A64_SP), ctx); + emit(A64_LDR64I(A64_R(0), A64_FP, 8), ctx); + emit_call((u64)bpf_unwind_fixup, ctx); + emit(A64_MOV(1, tmp, A64_R(0)), ctx); + emit(A64_POP(r0, r2, A64_SP), ctx); + build_epilogue(ctx, false, true); +} + +static const struct bpf_jit_arg_abi arm64_arg_abi = { + .nr_arg_regs = 8, + .even_reg_align = true, + .even_stack_align = true, +}; + static const u8 stack_arg_reg[] = { A64_R(5), A64_R(6), A64_R(7) }; #define NR_STACK_ARG_REGS ARRAY_SIZE(stack_arg_reg) @@ -1264,19 +1345,20 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct * kern_vm_start. A nullable arg preserves NULL by skipping the add, tested * on the truncated value as arena NULL is offset 0. */ -static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn) +static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct btf_func_model *fm) { const u8 arena_vm_base = bpf2a64[ARENA_VM_START]; - const struct btf_func_model *fm; - int i; + int i, slot; - fm = bpf_jit_find_kfunc_model(ctx->prog, insn); - if (!fm) - return -EINVAL; - - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { - const u8 reg = bpf2a64[BPF_REG_1 + i]; + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; + u8 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = bpf2a64[BPF_REG_1 + slot]; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; @@ -1295,6 +1377,52 @@ static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *ins return 0; } +static bool a64_arg_on_stack(u8 slot) +{ + return slot >= arm64_arg_abi.nr_arg_regs; +} + +static s32 a64_arg_stack_off(u8 slot) +{ + return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64); +} + +/* + * Move the arguments AAPCS64 places somewhere other than the argument slot the + * BPF calling convention gave them. Slot N is X(N) up to the eighth, and the + * outgoing stack argument area from SP beyond it, both for the slot an + * argument comes from and for the one it goes to. + * + * AAPCS64 only ever moves an argument to a higher slot, so no move here ever + * takes BPF_JIT_ARG_TMP: bpf_jit_plan_arg_moves() hands out the scratch only + * for a convention that moves one down, which needs a register to carry the + * value past its own destination. + */ +static void emit_kfunc_arg_moves(struct jit_ctx *ctx, const struct btf_func_model *fm) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 tmp = bpf2a64[TMP_REG_1]; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&arm64_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + + if (a64_arg_on_stack(src)) { + reg = tmp; + emit(A64_LDR64I(reg, A64_SP, a64_arg_stack_off(src)), ctx); + } else { + reg = src; + } + + if (a64_arg_on_stack(dst)) + emit(A64_STR64I(reg, A64_SP, a64_arg_stack_off(dst)), ctx); + else if (reg != dst) + emit(A64_MOV(1, dst, reg), ctx); + } +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -1713,9 +1841,15 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn if (ret < 0) return ret; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - ret = emit_kfunc_arena_args(ctx, insn); + const struct btf_func_model *fm; + + fm = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!fm) + return -EINVAL; + ret = emit_kfunc_arena_args(ctx, fm); if (ret < 0) return ret; + emit_kfunc_arg_moves(ctx, fm); } emit_call(func_addr, ctx); /* @@ -1727,13 +1861,34 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn emit(A64_MOV(1, r0, A64_R(0)), ctx); break; } + /* indirect call of a bpf subprog, dst holds its address */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * It's the same as a direct call of a subprog that is out of + * range of BL: the subprog starts with BTI JC, the arguments + * are in place, and the registers that hold the tail call + * counter and the private stack are callee saved. + */ + emit(A64_BLR(dst), ctx); + emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx); + break; /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(ctx)) return -EFAULT; break; + /* unwind_goto. The verifier expanded may_goto. */ + case BPF_JMP | BPF_JCOND: + ret = add_unwind_exentry(ctx, i + 1 + off); + if (ret) + return ret; + break; /* function return */ case BPF_JMP | BPF_EXIT: + if (bpf_is_unwind_exit(insn) && bpf_prog_can_unwind(ctx->prog)) { + emit_unwind_exit(ctx); + break; + } /* Optimization: when last instruction is EXIT, simply fallthrough to epilogue. */ if (i == ctx->prog->len - 1) @@ -2210,8 +2365,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr if (ctx.offset == NULL) goto out_off; - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + } out_cnt = bpf_out_stack_arg_cnt(env, prog); if (out_cnt) { @@ -2236,7 +2393,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr goto out_off; ctx.epilogue_offset = ctx.idx; - build_epilogue(&ctx, was_classic); + build_epilogue(&ctx, was_classic, false); build_plt(&ctx); extable_align = __alignof__(struct exception_table_entry); @@ -2293,7 +2450,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) goto out_free_hdr; - build_epilogue(&ctx, was_classic); + build_epilogue(&ctx, was_classic, false); build_plt(&ctx); /* Extra pass to validate JITed code. */ @@ -2385,6 +2542,21 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + +bool bpf_jit_supports_unwind(void) +{ + return true; +} + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &arm64_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; @@ -2413,6 +2585,16 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +bool bpf_jit_supports_callx(void) +{ + return true; +} + +bool bpf_jit_supports_large_stack(void) +{ + return true; +} + static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, struct bpf_tramp_node *node, int bargs_off, int retval_off, int run_ctx_off, bool save_ret) @@ -2528,33 +2710,41 @@ struct arg_aux { * arguments to be properly aligned) */ int ostack_for_args; + /* where AAPCS64 puts each argument slot: an argument register below + * the eighth, an on-stack argument slot from it up + */ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; }; static int calc_arg_aux(const struct btf_func_model *m, struct arg_aux *a) { - int stack_slots, nregs, slots, i; + int slots, i, slot, total; + + total = bpf_jit_place_args(&arm64_arg_abi, m, a->pos_of_slot); + if (total > MAX_BPF_FUNC_ARGS) + return -ENOTSUPP; /* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */ - for (i = 0, nregs = 0; i < m->nr_args; i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { slots = (m->arg_size[i] + 7) / 8; - if (nregs + slots <= 8) /* passed through register ? */ - nregs += slots; - else + if (a64_arg_on_stack(a->pos_of_slot[slot])) /* passed through register ? */ break; + slot += slots; } a->args_in_regs = i; - a->regs_for_args = nregs; + a->regs_for_args = slot; a->ostack_for_args = 0; a->bstack_for_args = 0; /* the rest arguments are passed through stack */ - for (; i < m->nr_args; i++) { - stack_slots = (m->arg_size[i] + 7) / 8; - a->bstack_for_args += stack_slots * 8; - a->ostack_for_args = a->ostack_for_args + stack_slots * 8; - } + for (; i < m->nr_args; i++) + a->bstack_for_args += ((m->arg_size[i] + 7) / 8) * 8; + + /* the outgoing area reaches the last slot, over any alignment hole */ + if (a->bstack_for_args) + a->ostack_for_args = a64_arg_stack_off(a->pos_of_slot[total - 1]) + 8; return 0; } @@ -2595,7 +2785,7 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, { u8 tmp = bpf2a64[TMP_REG_1]; u8 base_lo = bpf2a64[TMP_REG_2]; - int i, reg, doff, soff, slots; + int i, reg, slot, soff, slots; /* only the low 32 bits of the base take part in the subtraction */ if (arena_base) @@ -2604,12 +2794,13 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, /* store arguments to the stack for the bpf program, or restore * arguments from stack for the original function */ - for (i = 0, reg = 0; i < a->args_in_regs; i++) { + for (i = 0, slot = 0; i < a->args_in_regs; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; slots = (m->arg_size[i] + 7) / 8; while (slots-- > 0) { + reg = a->pos_of_slot[slot++]; if (for_call_origin) { emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); } else if (arena_arg) { @@ -2618,7 +2809,6 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, } else { emit(A64_STR64I(reg, A64_SP, bargs_off), ctx); } - reg++; bargs_off += 8; } } @@ -2630,9 +2820,11 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops * callback is called indirectly and only the FP/LR frame is saved, so * they start at FP + 16. + * + * The outgoing area mirrors the incoming one, hole and all; only the + * bpf program takes the arguments packed. */ soff = is_struct_ops ? 16 : 32; - doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ for (i = a->args_in_regs; i < m->nr_args; i++) { @@ -2642,7 +2834,9 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, slots = (m->arg_size[i] + 7) / 8; /* verifier ensures arg_size <= 16, so slots equals 1 or 2 */ while (slots-- > 0) { - emit(A64_LDR64I(tmp, A64_FP, soff), ctx); + int off = a64_arg_stack_off(a->pos_of_slot[slot++]); + + emit(A64_LDR64I(tmp, A64_FP, soff + off), ctx); /* if there is unused space in the last slot, clear * the garbage contained in the space. */ @@ -2657,19 +2851,21 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, */ if (arena_arg) emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo); - emit(A64_STR64I(tmp, A64_SP, doff), ctx); - soff += 8; - doff += 8; + if (for_call_origin) + emit(A64_STR64I(tmp, A64_SP, oargs_off + off), ctx); + else + emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx); + bargs_off += 8; } } } -static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) +static void restore_args(struct jit_ctx *ctx, int bargs_off, const struct arg_aux *a) { - int reg; + int slot; - for (reg = 0; reg < nregs; reg++) { - emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); + for (slot = 0; slot < a->regs_for_args; slot++) { + emit(A64_LDR64I(a->pos_of_slot[slot], A64_SP, bargs_off), ctx); bargs_off += 8; } } @@ -2937,7 +3133,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(ctx, bargs_off, a->regs_for_args); + restore_args(ctx, bargs_off, a); /* restore callee saved register x19 and x20 */ emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -3226,21 +3422,22 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, */ plt_target = (u64)&dummy_tramp; - if (plt_target) { - /* non-zero plt_target indicates we're patching a bpf prog, - * which is read only. - */ - if (set_memory_rw(PAGE_MASK & ((uintptr_t)&plt->target), 1)) - return -EFAULT; - WRITE_ONCE(plt->target, plt_target); - set_memory_ro(PAGE_MASK & ((uintptr_t)&plt->target), 1); - /* since plt target points to either the new trampoline - * or dummy_tramp, even if another CPU reads the old plt - * target value before fetching the bl instruction to plt, - * it will be brought back by dummy_tramp, so no barrier is - * required here. - */ - } + /* + * non-zero plt_target indicates we're patching a bpf prog, + * which is read only. The prog may share its page in the bpf + * prog pack with other progs, so write the aligned target + * through the text patching fixmap without flipping the + * page permissions to avoid racing with concurrent pokers. + * + * since plt target points to either the new trampoline + * or dummy_tramp, even if another CPU reads the old plt + * target value before fetching the bl instruction to plt, + * it will be brought back by dummy_tramp, so no barrier is + * required here. + */ + if (plt_target && + aarch64_insn_write_literal_u64(&plt->target, plt_target)) + return -EFAULT; /* if the old target and the new target are both long jumps, no * patching is required @@ -3305,6 +3502,11 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index b15a59b..997c62a 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg
@@ -401,6 +401,57 @@ Field 0 SO EndSysreg +SysregFields SPSR_ELx +Res0 63:37 +Field 36 UINJ +Field 35 PACM +Field 34 EXLOCK +Res0 33 +Field 32 PM +Field 31 N +Field 30 Z +Field 29 C +Field 28 V +Res0 27:26 +Field 25 TCO +Field 24 DIT +Field 23 UAO +Field 22 PAN +Field 21 SS +Field 20 IL +Res0 19:14 +Field 13 ALLINT +Field 12 SSBS +Field 11:10 BTYPE +Field 9 D +Field 8 A +Field 7 I +Field 6 F +Res0 5 +Field 4 M4 +Enum 3:0 M + 0b0000 EL0 + 0b0100 EL1t + 0b0101 EL1h + 0b1000 EL2t + 0b1001 EL2h + 0b1100 EL3t + 0b1101 EL3h +EndEnum +EndSysregFields + +Sysreg SPSR_EL1 3 0 4 0 0 +Fields SPSR_ELx +EndSysreg + +Sysreg SPSR_EL2 3 4 4 0 0 +Fields SPSR_ELx +EndSysreg + +Sysreg SPSR_EL12 3 5 4 0 0 +Mapping SPSR_EL1 +EndSysreg + Sysreg ID_PFR0_EL1 3 0 0 1 0 Res0 63:32 UnsignedEnum 31:28 RAS
diff --git a/arch/hexagon/include/asm/cmpxchg.h b/arch/hexagon/include/asm/cmpxchg.h index 9c58fb8..e6114e7 100644 --- a/arch/hexagon/include/asm/cmpxchg.h +++ b/arch/hexagon/include/asm/cmpxchg.h
@@ -8,6 +8,8 @@ #ifndef _ASM_CMPXCHG_H #define _ASM_CMPXCHG_H +#include <asm-generic/cmpxchg-local.h> + /* * __arch_xchg - atomically exchange a register and a memory location * @x: value to swap @@ -72,4 +74,8 @@ __arch_xchg(unsigned long x, volatile void *ptr, int size) __oldval; \ }) +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), \ + (unsigned long)(o), (unsigned long)(n), sizeof(*(ptr)))) + #endif /* _ASM_CMPXCHG_H */
diff --git a/arch/loongarch/include/asm/kvm_dmsintc.h b/arch/loongarch/include/asm/kvm_dmsintc.h index 7c01587..e19b3f5 100644 --- a/arch/loongarch/include/asm/kvm_dmsintc.h +++ b/arch/loongarch/include/asm/kvm_dmsintc.h
@@ -22,7 +22,6 @@ struct dmsintc_state { int kvm_loongarch_register_dmsintc_device(void); void kvm_loongarch_unregister_dmsintc_device(void); void dmsintc_inject_irq(struct kvm_vcpu *vcpu); -int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level); -int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 vector, int level); +int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data); #endif
diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 65d91c3..b0692f8 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h
@@ -26,6 +26,8 @@ #include <asm/loongarch.h> #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM /* Loongarch KVM register ids */ #define KVM_GET_IOC_CSR_IDX(id) ((id & KVM_CSR_IDX_MASK) >> LOONGARCH_REG_SHIFT) @@ -79,6 +81,7 @@ struct kvm_arch_memory_slot { #define HOST_MAX_PMNUM 16 struct kvm_context { unsigned long vpid_cache; + unsigned long vmid_cache; struct kvm_vcpu *last_vcpu; /* Host PMU CSR */ u64 perf_ctrl[HOST_MAX_PMNUM]; @@ -133,6 +136,8 @@ struct kvm_arch { unsigned long kvm_features; s64 time_offset; + cpumask_t tlb_flush_pending; + unsigned long vmid[NR_CPUS]; struct kvm_context __percpu *vmcs; struct loongarch_ipi *ipi; struct loongarch_dmsintc *dmsintc; @@ -164,11 +169,11 @@ enum emulation_result { #define KVM_LARCH_SWCSR_LATEST (0x1 << 3) #define KVM_LARCH_HWCSR_USABLE (0x1 << 4) -#define LOONGARCH_PV_FEAT_UPDATED BIT_ULL(63) #define LOONGARCH_PV_FEAT_MASK (BIT(KVM_FEATURE_IPI) | \ BIT(KVM_FEATURE_PREEMPT) | \ BIT(KVM_FEATURE_STEAL_TIME) | \ BIT(KVM_FEATURE_USER_HCALL) | \ + BIT(KVM_FEATURE_CRASH) | \ BIT(KVM_FEATURE_VIRT_EXTIOI)) struct kvm_vcpu_arch { @@ -233,14 +238,16 @@ struct kvm_vcpu_arch { /* Cache for pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - /* vcpu's vpid */ - u64 vpid; + /* vCPU's vpid & tgid */ + unsigned long vpid, tgid; gpa_t flush_gpa; /* Frequency of stable timer in Hz */ u64 timer_mhz; ktime_t expire; + /* vCPU has run at least once */ + bool has_run; /* Last CPU the vCPU state was loaded on */ int last_sched_cpu; /* mp state */ @@ -318,6 +325,8 @@ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); /* MMU handling */ void kvm_flush_tlb_all(void); +void kvm_flush_tlb_all_stage1(void); +void kvm_flush_tlb_all_stage2(void); void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa); int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write, int ecode); @@ -349,6 +358,8 @@ static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS +int kvm_arch_flush_remote_tlbs(struct kvm *kvm); void kvm_check_vpid(struct kvm_vcpu *vcpu); enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer); void kvm_init_vmcs(struct kvm *kvm);
diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h index fb17ba0..2f4b9cc 100644 --- a/arch/loongarch/include/asm/kvm_para.h +++ b/arch/loongarch/include/asm/kvm_para.h
@@ -14,6 +14,7 @@ #define KVM_HCALL_CODE_SERVICE 0 #define KVM_HCALL_CODE_SWDBG 1 #define KVM_HCALL_CODE_USER_SERVICE 2 +#define KVM_HCALL_CODE_CRASH 3 #define KVM_HCALL_SERVICE HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_SERVICE) #define KVM_HCALL_FUNC_IPI 1 @@ -23,6 +24,8 @@ #define KVM_HCALL_USER_SERVICE HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_USER_SERVICE) +#define KVM_HCALL_CRASH HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_CRASH) + /* * LoongArch hypercall return code */
diff --git a/arch/loongarch/include/asm/kvm_pch_pic.h b/arch/loongarch/include/asm/kvm_pch_pic.h index 887b043..19ed149 100644 --- a/arch/loongarch/include/asm/kvm_pch_pic.h +++ b/arch/loongarch/include/asm/kvm_pch_pic.h
@@ -72,6 +72,6 @@ struct kvm_kernel_irq_routing_entry; int kvm_loongarch_register_pch_pic_device(void); void kvm_loongarch_unregister_pch_pic_device(void); void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level); -int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level); +int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e); #endif /* __ASM_KVM_PCH_PIC_H */
diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 2a6bc99..88e3bdf 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h
@@ -138,6 +138,9 @@ #define CPUCFG6_PMBITS_SHIFT 8 #define CPUCFG6_UPM BIT(14) +#define LOONGARCH_CPUCFG8 0x8 +#define CPUCFG8_S2TGID BIT(8) + #define LOONGARCH_CPUCFG16 0x10 #define CPUCFG16_L1_IUPRE BIT(0) #define CPUCFG16_L1_IUUNIFY BIT(1) @@ -350,6 +353,8 @@ #define CSR_GTLBC_TGID_WIDTH 8 #define CSR_GTLBC_TGID_SHIFT_END (CSR_GTLBC_TGID_SHIFT + CSR_GTLBC_TGID_WIDTH - 1) #define CSR_GTLBC_TGID (_ULCAST_(0xff) << CSR_GTLBC_TGID_SHIFT) +#define CSR_GTLBC_USES2TGID_SHIFT 14 +#define CSR_GTLBC_USES2TGID (_ULCAST_(0x1) << CSR_GTLBC_USES2TGID_SHIFT) #define CSR_GTLBC_TOTI_SHIFT 13 #define CSR_GTLBC_TOTI (_ULCAST_(0x1) << CSR_GTLBC_TOTI_SHIFT) #define CSR_GTLBC_USETGID_SHIFT 12
diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h index cd0b5c1..b222bfc 100644 --- a/arch/loongarch/include/uapi/asm/kvm.h +++ b/arch/loongarch/include/uapi/asm/kvm.h
@@ -106,6 +106,7 @@ struct kvm_fpu { #define KVM_LOONGARCH_VM_FEAT_PTW 8 #define KVM_LOONGARCH_VM_FEAT_MSGINT 9 #define KVM_LOONGARCH_VM_FEAT_PV_PREEMPT 10 +#define KVM_LOONGARCH_VM_FEAT_CRASH 11 /* Device Control API on vcpu fd */ #define KVM_LOONGARCH_VCPU_CPUCFG 0
diff --git a/arch/loongarch/include/uapi/asm/kvm_para.h b/arch/loongarch/include/uapi/asm/kvm_para.h index d28cbca..bf1a0c8 100644 --- a/arch/loongarch/include/uapi/asm/kvm_para.h +++ b/arch/loongarch/include/uapi/asm/kvm_para.h
@@ -16,6 +16,7 @@ #define KVM_FEATURE_IPI 1 #define KVM_FEATURE_STEAL_TIME 2 #define KVM_FEATURE_PREEMPT 3 +#define KVM_FEATURE_CRASH 4 /* BIT 24 - 31 are features configurable by user space vmm */ #define KVM_FEATURE_VIRT_EXTIOI 24 #define KVM_FEATURE_USER_HCALL 25
diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 1b861cb..7afeff8 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c
@@ -301,6 +301,7 @@ static void __used output_kvm_defines(void) OFFSET(KVM_ARCH_HTP, kvm_vcpu_arch, host_tp); OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); OFFSET(KVM_ARCH_KVMPGD, kvm_vcpu_arch, kvm_pgd); + OFFSET(KVM_ARCH_TGID, kvm_vcpu_arch, tgid); OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry);
diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 74d31f2..7121e74 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c
@@ -241,6 +241,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) if (config & CPUCFG6_PMP) c->options |= LOONGARCH_CPU_PMP; + config = read_cpucfg(LOONGARCH_CPUCFG8); + if (config & CPUCFG8_S2TGID) + c->options |= LOONGARCH_CPU_GUESTID; + config = csr_read32(LOONGARCH_CSR_ASID); config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT; asid_mask = GENMASK(config - 1, 0);
diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index 15da2d8..fd38732 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig
@@ -23,12 +23,9 @@ select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQCHIP - select HAVE_KVM_MSI - select HAVE_KVM_READONLY_MEM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING - select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS
diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 4f58e6e..3b6daa9 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c
@@ -911,6 +911,17 @@ static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode) vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE; ret = RESUME_HOST; break; + case KVM_HCALL_CRASH: + if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_CRASH)) { + kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); + break; + } + + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; + vcpu->run->system_event.ndata = 0; + ret = RESUME_HOST; + break; case KVM_HCALL_SWDBG: /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */ if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) {
diff --git a/arch/loongarch/kvm/intc/dmsintc.c b/arch/loongarch/kvm/intc/dmsintc.c index 89f980d..ff327b4 100644 --- a/arch/loongarch/kvm/intc/dmsintc.c +++ b/arch/loongarch/kvm/intc/dmsintc.c
@@ -14,9 +14,6 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) unsigned long vector[4], old; struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - if (!ds) - return; - for (i = 0; i < 4; i++) { old = atomic64_read(&(ds->vector_map[i])); vector[i] = old ? atomic64_xchg(&(ds->vector_map[i]), 0) : 0; @@ -43,42 +40,25 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) } } -int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, - struct kvm_vcpu *vcpu, u32 vector, int level) -{ - struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - - if (!level) - return 0; - if (!vcpu || vector >= 256) - return -EINVAL; - if (!ds) - return -ENODEV; - - if (!kvm_guest_has_msgint(&vcpu->arch)) - return -EINVAL; - - set_bit(vector, (unsigned long *)&ds->vector_map); - kvm_queue_irq(vcpu, INT_AVEC); - kvm_vcpu_kick(vcpu); - - return 0; -} - -int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level) +int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data) { unsigned int irq, cpu; struct kvm_vcpu *vcpu; irq = (addr >> AVEC_IRQ_SHIFT) & AVEC_IRQ_MASK; cpu = (addr >> AVEC_CPU_SHIFT) & kvm->arch.dmsintc->cpu_mask; - if (cpu >= KVM_MAX_VCPUS) - return -EINVAL; vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); if (!vcpu) return -EINVAL; - return dmsintc_deliver_msi_to_vcpu(kvm, vcpu, irq, level); + if (!kvm_guest_has_msgint(&vcpu->arch)) + return -EINVAL; + + set_bit(irq, (unsigned long *)&vcpu->arch.dmsintc_state.vector_map); + kvm_queue_irq(vcpu, INT_AVEC); + kvm_vcpu_kick(vcpu); + + return 0; } static int kvm_dmsintc_ctrl_access(struct kvm_device *dev,
diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c index 7b333a4..80faa77 100644 --- a/arch/loongarch/kvm/intc/ipi.c +++ b/arch/loongarch/kvm/intc/ipi.c
@@ -423,15 +423,12 @@ static int kvm_ipi_create(struct kvm_device *dev, u32 type) mutex_unlock(&kvm->slots_lock); if (ret < 0) { kvm_pr_unimpl("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); - goto err; + kfree(s); + return ret; } kvm->arch.ipi = s; return 0; - -err: - kfree(s); - return -EFAULT; } static void kvm_ipi_destroy(struct kvm_device *dev)
diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c index 2b63b0c..220fc23 100644 --- a/arch/loongarch/kvm/intc/pch_pic.c +++ b/arch/loongarch/kvm/intc/pch_pic.c
@@ -68,20 +68,20 @@ void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level) } /* msi irq handler */ -int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level) +int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e) { u64 msg_addr = (((u64)e->msi.address_hi) << 32) | e->msi.address_lo; if (cpu_has_msgint && kvm->arch.dmsintc && msg_addr >= kvm->arch.dmsintc->msg_addr_base && msg_addr < (kvm->arch.dmsintc->msg_addr_base + kvm->arch.dmsintc->msg_addr_size)) { - return dmsintc_set_irq(kvm, msg_addr, e->msi.data, level); + return dmsintc_set_irq(kvm, msg_addr, e->msi.data); } if (e->msi.data >= EIOINTC_IRQS) return -EINVAL; - eiointc_set_irq(kvm->arch.eiointc, e->msi.data, level); + eiointc_set_irq(kvm->arch.eiointc, e->msi.data, 1); return 0; }
diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c index 40ed108..2e9c320 100644 --- a/arch/loongarch/kvm/irqfd.c +++ b/arch/loongarch/kvm/irqfd.c
@@ -29,7 +29,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, if (!level) return -1; - return pch_msi_set_irq(kvm, e, level); + return pch_msi_set_irq(kvm, e); } /* @@ -78,7 +78,7 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, pch_pic_set_irq(kvm->arch.pch_pic, e->irqchip.pin, level); return 0; case KVM_IRQ_ROUTING_MSI: - return pch_msi_set_irq(kvm, e, level); + return pch_msi_set_irq(kvm, e); default: return -EWOULDBLOCK; }
diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 236523d..c65ae2f 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c
@@ -218,14 +218,17 @@ static void kvm_update_vpid(struct kvm_vcpu *vcpu, int cpu) ++vpid; /* vpid 0 reserved for root */ /* start new vpid cycle */ - kvm_flush_tlb_all(); + if (!cpu_has_guestid) + kvm_flush_tlb_all(); + else + kvm_flush_tlb_all_stage1(); } context->vpid_cache = vpid; vcpu->arch.vpid = vpid; } -void kvm_check_vpid(struct kvm_vcpu *vcpu) +static void __kvm_check_vpid(struct kvm_vcpu *vcpu) { int cpu; bool migrated; @@ -253,7 +256,6 @@ void kvm_check_vpid(struct kvm_vcpu *vcpu) kvm_update_vpid(vcpu, cpu); trace_kvm_vpid_change(vcpu, vcpu->arch.vpid); vcpu->cpu = cpu; - kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); /* * LLBCTL is a separated guest CSR register from host, a general @@ -279,6 +281,62 @@ void kvm_check_vpid(struct kvm_vcpu *vcpu) } } +static void kvm_update_vmid(struct kvm_vcpu *vcpu, int cpu) +{ + unsigned long vmid; + struct kvm_context *context; + + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + vmid = context->vmid_cache + 1; + if (!(vmid & vpid_mask)) { + /* finish round of vmid loop */ + if (unlikely(!vmid)) + vmid = vpid_mask + 1; + + ++vmid; /* vmid 0 reserved for root */ + + /* start new vmid cycle */ + kvm_flush_tlb_all_stage2(); + } + + context->vmid_cache = vmid; + vcpu->kvm->arch.vmid[cpu] = vmid; +} + +static void __kvm_check_vmid(struct kvm_vcpu *vcpu) +{ + int cpu; + unsigned long ver, old, vmid; + + /* On CPUs like Loongson-3A5000, vmid must be the same as vpid */ + if (!cpu_has_guestid) { + vmid = vcpu->arch.vpid & vpid_mask; + if (vcpu->arch.tgid != vmid) { + vcpu->arch.tgid = vcpu->arch.vpid & vpid_mask; + kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } + return; + } + + cpu = smp_processor_id(); + if (cpumask_test_and_clear_cpu(cpu, &vcpu->kvm->arch.tlb_flush_pending)) + vcpu->kvm->arch.vmid[cpu] = 0; + + /* Check if our vmid is of an older version */ + ver = vcpu->kvm->arch.vmid[cpu] & ~vpid_mask; + old = this_cpu_ptr(vcpu->kvm->arch.vmcs)->vmid_cache & ~vpid_mask; + if (ver != old) + kvm_update_vmid(vcpu, cpu); + + vcpu->arch.tgid = vcpu->kvm->arch.vmid[cpu] & vpid_mask; +} + +void kvm_check_vpid(struct kvm_vcpu *vcpu) +{ + __kvm_check_vpid(vcpu); + __kvm_check_vmid(vcpu); +} + void kvm_init_vmcs(struct kvm *kvm) { kvm->arch.vmcs = vmcs; @@ -300,7 +358,7 @@ int kvm_arch_enable_virtualization_cpu(void) write_csr_gcfg(0); write_csr_gstat(0); write_csr_gintc(0); - clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI); + clear_csr_gtlbc(CSR_GTLBC_TOTI | CSR_GTLBC_USETGID | CSR_GTLBC_USES2TGID); /* * Enable virtualization features granting guest direct control of @@ -323,6 +381,8 @@ int kvm_arch_enable_virtualization_cpu(void) /* Enable using TGID */ set_csr_gtlbc(CSR_GTLBC_USETGID); + if (cpu_has_guestid) + set_csr_gtlbc(CSR_GTLBC_USES2TGID); kvm_debug("GCFG:%lx GSTAT:%lx GINTC:%lx GTLBC:%lx", read_csr_gcfg(), read_csr_gstat(), read_csr_gintc(), read_csr_gtlbc()); @@ -341,7 +401,7 @@ void kvm_arch_disable_virtualization_cpu(void) write_csr_gcfg(0); write_csr_gstat(0); write_csr_gintc(0); - clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI); + clear_csr_gtlbc(CSR_GTLBC_TOTI | CSR_GTLBC_USETGID | CSR_GTLBC_USES2TGID); /* Flush any remaining guest TLB entries */ kvm_flush_tlb_all(); @@ -376,6 +436,7 @@ static int kvm_loongarch_env_init(void) for_each_possible_cpu(cpu) { context = per_cpu_ptr(vmcs, cpu); context->vpid_cache = vpid_mask + 1; + context->vmid_cache = vpid_mask + 1; context->last_vcpu = NULL; }
diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index 3e9a0b28..64fddd9 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c
@@ -903,7 +903,12 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, bool write) * there is invalid tlb with small page * need flush these invalid tlbs for current vcpu */ - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + if (!cpu_has_ptw) { + if (!cpu_has_guestid) + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + else + cpumask_set_cpu(vcpu->cpu, &vcpu->kvm->arch.tlb_flush_pending); + } ++kvm->stat.hugepages; } else if (kvm_pte_huge(*ptep) && write) ptep = kvm_split_huge(vcpu, ptep, gfn); @@ -939,8 +944,11 @@ int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write, in * * With SW PTW, invalid TLB is added in TLB refill exception. */ - vcpu->arch.flush_gpa = gpa; - kvm_make_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + if (!cpu_has_guestid) { + vcpu->arch.flush_gpa = gpa; + kvm_make_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } else + cpumask_set_cpu(vcpu->cpu, &vcpu->kvm->arch.tlb_flush_pending); } return 0; @@ -949,3 +957,14 @@ int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write, in void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { } + +int kvm_arch_flush_remote_tlbs(struct kvm *kvm) +{ + /* Queue a TLB flush for each CPU to perform on next vCPU loading */ + if (cpu_has_guestid) { + cpumask_setall(&kvm->arch.tlb_flush_pending); + smp_wmb(); /* Be sure that other CPUS can watch the changes */ + } + + return 1; /* Return 1 continue to send IPI to running vCPUs */ +}
diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index 936e4ae..e17e9ed 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S
@@ -66,9 +66,8 @@ ld.d t0, a2, KVM_ARCH_KVMPGD csrwr t0, LOONGARCH_CSR_PGDL - /* Mix GID and RID */ - csrrd t1, LOONGARCH_CSR_GSTAT - bstrpick.w t1, t1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT + /* Set VMID for gpa --> hpa mapping */ + ld.d t1, a2, KVM_ARCH_TGID csrrd t0, LOONGARCH_CSR_GTLBC bstrins.w t0, t1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT csrwr t0, LOONGARCH_CSR_GTLBC
diff --git a/arch/loongarch/kvm/tlb.c b/arch/loongarch/kvm/tlb.c index ebdbe92..994216c 100644 --- a/arch/loongarch/kvm/tlb.c +++ b/arch/loongarch/kvm/tlb.c
@@ -21,9 +21,26 @@ void kvm_flush_tlb_all(void) local_irq_restore(flags); } -void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) +/* Invalidate all stage1 TLB entries including GVA-->GPA mappings */ +void kvm_flush_tlb_all_stage1(void) { lockdep_assert_irqs_disabled(); + invtlb_all(INVGTLB_ALLGID_GVA_TO_GPA, 0, 0); +} + +/* Invalidate all stage2 TLB entries including GPA-->HPA mappings */ +void kvm_flush_tlb_all_stage2(void) +{ + lockdep_assert_irqs_disabled(); + invtlb_all(INVTLB_ALLGID_GPA_TO_HPA, 0, 0); +} + +void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) +{ + unsigned int vmid; + + lockdep_assert_irqs_disabled(); gpa &= (PAGE_MASK << 1); - invtlb(INVTLB_GID_ADDR, read_csr_gstat() & CSR_GSTAT_GID, gpa); + vmid = (vcpu->arch.tgid << CSR_GSTAT_GID_SHIFT) & CSR_GSTAT_GID; + invtlb(INVTLB_GID_ADDR, vmid, gpa); }
diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 8e028be..4026b9b 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c
@@ -1166,12 +1166,17 @@ static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, /* All vCPUs need set the same PV features */ spin_lock(&kvm->arch.pv_setting_lock); - if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) - && ((kvm->arch.pv_features & valid) != val)) { + if ((kvm->arch.pv_features & valid) == val) { + spin_unlock(&kvm->arch.pv_setting_lock); + return 0; + } + + if (vcpu->arch.has_run) { spin_unlock(&kvm->arch.pv_setting_lock); return -EINVAL; } - kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; + + kvm->arch.pv_features = val; spin_unlock(&kvm->arch.pv_setting_lock); return 0; default: @@ -1858,6 +1863,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) int r = -EINTR; struct kvm_run *run = vcpu->run; + /* Mark vCPU has run at least once */ + if (!vcpu->arch.has_run) + vcpu->arch.has_run = true; + if (vcpu->mmio_needed) { if (!vcpu->mmio_is_write) kvm_complete_mmio_read(vcpu, run);
diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 6dabb22..e4c1c65 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c
@@ -50,7 +50,9 @@ static void kvm_vm_init_features(struct kvm *kvm) /* Enable all PV features by default */ kvm->arch.pv_features |= BIT(KVM_FEATURE_IPI); + kvm->arch.pv_features |= BIT(KVM_FEATURE_CRASH); kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI); + kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_CRASH); if (kvm_pvtime_supported()) { kvm->arch.pv_features |= BIT(KVM_FEATURE_PREEMPT); kvm->arch.pv_features |= BIT(KVM_FEATURE_STEAL_TIME);
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index e78eb58..9393e4b 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c
@@ -2245,8 +2245,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); if (ctx.offset == NULL)
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index fefcaa5..e2d474e 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig
@@ -579,7 +579,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 84d4a6e..77669e3 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig
@@ -533,7 +533,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 22d8a85..43ed1e3 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig
@@ -558,7 +558,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 0f79c63..44c1a22 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig
@@ -525,7 +525,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 2bf412a..7ab2537 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig
@@ -535,7 +535,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 5496f1b..13f6428 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig
@@ -554,7 +554,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index a8f593b..1bbd2aa 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig
@@ -644,7 +644,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index bbce43b..3bfc36f 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig
@@ -525,7 +525,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index a63d09d..41ceed8 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig
@@ -526,7 +526,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index d6b7cba..af4b5c6 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig
@@ -544,7 +544,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index bee5cdd..1140951 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig
@@ -522,7 +522,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index ef2b76a..bc9358e 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig
@@ -523,7 +523,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index c14b108..d481275 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h
@@ -893,5 +893,6 @@ static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS +#define __KVM_HAVE_READONLY_MEM #endif /* __MIPS_KVM_HOST_H__ */
diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index b43bfd4..adea35d 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h
@@ -85,15 +85,20 @@ Ip_u1u2(_ctc1); Ip_u2u1(_ctcmsa); Ip_u2u1s3(_daddiu); Ip_u3u1u2(_daddu); +Ip_u1u2(_ddiv); +Ip_u3u1u2(_ddiv_r6); Ip_u1u2(_ddivu); Ip_u3u1u2(_ddivu_r6); Ip_u1(_di); Ip_u2u1msbu3(_dins); Ip_u2u1msbu3(_dinsm); Ip_u2u1msbu3(_dinsu); +Ip_u1u2(_div); +Ip_u3u1u2(_div_r6); Ip_u1u2(_divu); Ip_u3u1u2(_divu_r6); Ip_u1u2u3(_dmfc0); +Ip_u3u1u2(_dmod); Ip_u3u1u2(_dmodu); Ip_u1u2u3(_dmtc0); Ip_u1u2(_dmultu); @@ -135,6 +140,7 @@ Ip_u1u2u3(_mfc0); Ip_u1u2u3(_mfhc0); Ip_u1(_mfhi); Ip_u1(_mflo); +Ip_u3u1u2(_mod); Ip_u3u1u2(_modu); Ip_u3u1u2(_movn); Ip_u3u1u2(_movz); @@ -156,6 +162,8 @@ Ip_u2s3u1(_sb); Ip_u2s3u1(_sc); Ip_u2s3u1(_scd); Ip_u2s3u1(_sd); +Ip_u2u1(_seb); +Ip_u2u1(_seh); Ip_u3u1u2(_seleqz); Ip_u3u1u2(_selnez); Ip_u2s3u1(_sh);
diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index b1b9a1d6..849ce7c 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig
@@ -22,9 +22,7 @@ select EXPORT_UASM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT - select KVM_MMIO select KVM_GENERIC_HARDWARE_ENABLING - select HAVE_KVM_READONLY_MEM help Support for hosting Guest kernels.
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index e15c670..2d1668b 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c
@@ -75,6 +75,9 @@ static const struct insn insn_table[insn_invalid] = { [insn_ctcmsa] = {M(msa_op, 0, msa_ctc_op, 0, 0, msa_elm_op), RD | RE}, [insn_daddiu] = {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, [insn_daddu] = {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD}, + [insn_ddiv] = {M(spec_op, 0, 0, 0, 0, ddiv_op), RS | RT}, + [insn_ddiv_r6] = {M(spec_op, 0, 0, 0, ddiv_ddiv6_op, ddiv_op), + RS | RT | RD}, [insn_ddivu] = {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT}, [insn_ddivu_r6] = {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op), RS | RT | RD}, @@ -82,10 +85,15 @@ static const struct insn insn_table[insn_invalid] = { [insn_dins] = {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE}, [insn_dinsm] = {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE}, [insn_dinsu] = {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE}, + [insn_div] = {M(spec_op, 0, 0, 0, 0, div_op), RS | RT}, + [insn_div_r6] = {M(spec_op, 0, 0, 0, div_div6_op, div_op), + RS | RT | RD}, [insn_divu] = {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT}, [insn_divu_r6] = {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op), RS | RT | RD}, [insn_dmfc0] = {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmod] = {M(spec_op, 0, 0, 0, ddiv_dmod_op, ddiv_op), + RS | RT | RD}, [insn_dmodu] = {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op), RS | RT | RD}, [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, @@ -140,6 +148,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_mfhc0] = {M(cop0_op, mfhc0_op, 0, 0, 0, 0), RT | RD | SET}, [insn_mfhi] = {M(spec_op, 0, 0, 0, 0, mfhi_op), RD}, [insn_mflo] = {M(spec_op, 0, 0, 0, 0, mflo_op), RD}, + [insn_mod] = {M(spec_op, 0, 0, 0, div_mod_op, div_op), + RS | RT | RD}, [insn_modu] = {M(spec_op, 0, 0, 0, divu_modu_op, divu_op), RS | RT | RD}, [insn_movn] = {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD}, @@ -177,6 +187,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_scd] = {M6(spec3_op, 0, 0, 0, scd6_op), RS | RT | SIMM9}, #endif [insn_sd] = {M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, + [insn_seb] = {M(spec3_op, 0, 0, 0, seb_op, bshfl_op), RT | RD}, + [insn_seh] = {M(spec3_op, 0, 0, 0, seh_op, bshfl_op), RT | RD}, [insn_seleqz] = {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD}, [insn_selnez] = {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD}, [insn_sh] = {M(sh_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c index 1251409..139afb0 100644 --- a/arch/mips/mm/uasm.c +++ b/arch/mips/mm/uasm.c
@@ -49,19 +49,21 @@ enum opcode { insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1, insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez, insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1, - insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu, - insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu, - insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu, + insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddiv, + insn_ddiv_r6, insn_ddivu, insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, + insn_dinsu, insn_div, insn_div_r6, insn_divu, insn_divu_r6, insn_dmfc0, + insn_dmod, insn_dmodu, insn_dmtc0, insn_dmultu, insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll, insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl, insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins, insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld, insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, - insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, - insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, - insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, - insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_mflo, insn_mod, insn_modu, insn_movn, insn_movz, insn_mtc0, + insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, + insn_muhu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, + insn_sb, insn_sc, insn_scd, insn_seb, insn_seh, + insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, @@ -287,9 +289,12 @@ I_u1u2(_cfc1) I_u2u1(_cfcmsa) I_u1u2(_ctc1) I_u2u1(_ctcmsa) +I_u1u2(_ddiv) +I_u3u1u2(_ddiv_r6) I_u1u2(_ddivu) I_u3u1u2(_ddivu_r6) I_u1u2u3(_dmfc0) +I_u3u1u2(_dmod) I_u3u1u2(_dmodu) I_u1u2u3(_dmtc0) I_u1u2(_dmultu) @@ -297,6 +302,8 @@ I_u3u1u2(_dmulu) I_u2u1s3(_daddiu) I_u3u1u2(_daddu) I_u1(_di); +I_u1u2(_div) +I_u3u1u2(_div_r6) I_u1u2(_divu) I_u3u1u2(_divu_r6) I_u2u1(_dsbh); @@ -332,6 +339,7 @@ I_u2s3u1(_lw) I_u2s3u1(_lwu) I_u1u2u3(_mfc0) I_u1u2u3(_mfhc0) +I_u3u1u2(_mod) I_u3u1u2(_modu) I_u3u1u2(_movn) I_u3u1u2(_movz) @@ -353,6 +361,8 @@ I_u2s3u1(_sb) I_u2s3u1(_sc) I_u2s3u1(_scd) I_u2s3u1(_sd) +I_u2u1(_seb) +I_u2u1(_seh) I_u3u1u2(_seleqz) I_u3u1u2(_selnez) I_u2s3u1(_sh)
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index 6ee4abe..4682dbb 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c
@@ -208,7 +208,7 @@ void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) } /* Validate ALU immediate range */ -bool valid_alu_i(u8 op, s32 imm) +bool valid_alu_i(u8 op, s32 imm, s16 off) { switch (BPF_OP(op)) { case BPF_NEG: @@ -237,6 +237,9 @@ bool valid_alu_i(u8 op, s32 imm) return imm == 0 || (imm > 0 && is_power_of_2(imm)); case BPF_DIV: case BPF_MOD: + /* Do not use unsigned shift/mask rewrites for signed div/mod. */ + if (off == 1) + return false; /* imm must be an 17-bit power of two */ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); } @@ -338,8 +341,42 @@ void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) clobber_reg(ctx, dst); } +/* ALU division operation (32-bit) */ +static void emit_div(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips32r6) { + if (off == 1) + emit(ctx, div_r6, dst, dst, src); + else + emit(ctx, divu_r6, dst, dst, src); + } else { + if (off == 1) + emit(ctx, div, dst, src); + else + emit(ctx, divu, dst, src); + emit(ctx, mflo, dst); + } +} + +/* ALU modulo operation (32-bit) */ +static void emit_mod(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips32r6) { + if (off == 1) + emit(ctx, mod, dst, dst, src); + else + emit(ctx, modu, dst, dst, src); + } else { + if (off == 1) + emit(ctx, div, dst, src); + else + emit(ctx, divu, dst, src); + emit(ctx, mfhi, dst); + } +} + /* ALU register operation (32-bit) */ -void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) +void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off) { switch (BPF_OP(op)) { /* dst = dst & src */ @@ -385,21 +422,11 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - if (cpu_has_mips32r6) { - emit(ctx, divu_r6, dst, dst, src); - } else { - emit(ctx, divu, dst, src); - emit(ctx, mflo, dst); - } + emit_div(ctx, dst, src, off); break; /* dst = dst % src */ case BPF_MOD: - if (cpu_has_mips32r6) { - emit(ctx, modu, dst, dst, src); - } else { - emit(ctx, divu, dst, src); - emit(ctx, mfhi, dst); - } + emit_mod(ctx, dst, src, off); break; } clobber_reg(ctx, dst); @@ -823,7 +850,7 @@ void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) } /* Jump always */ -int emit_ja(struct jit_context *ctx, s16 off) +int emit_ja(struct jit_context *ctx, s32 off) { int target = get_target(ctx, ctx->bpf_index + off + 1);
diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h index a37fe20..d0ab5c9 100644 --- a/arch/mips/net/bpf_jit_comp.h +++ b/arch/mips/net/bpf_jit_comp.h
@@ -163,7 +163,7 @@ void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); /* Validate ALU/ALU64 immediate range */ -bool valid_alu_i(u8 op, s32 imm); +bool valid_alu_i(u8 op, s32 imm, s16 off); /* Rewrite ALU/ALU64 immediate operation */ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); @@ -172,7 +172,7 @@ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); /* ALU register operation (32-bit) */ -void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); +void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off); /* Atomic read-modify-write (32-bit) */ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); @@ -204,7 +204,7 @@ void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); /* Jump always */ -int emit_ja(struct jit_context *ctx, s16 off); +int emit_ja(struct jit_context *ctx, s32 off); /* Jump to epilogue */ int emit_exit(struct jit_context *ctx);
diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 15a2a15..84d9c70 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c
@@ -190,6 +190,69 @@ static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) } } +/* Sign-extend an 8- or 16-bit field into a native register. */ +static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + int shift; + + if (cpu_has_mips32r2 || cpu_has_mips32r6) { + if (off == 8) + emit(ctx, seb, dst, src); + else + emit(ctx, seh, dst, src); + } else { + shift = 32 - off; + emit(ctx, sll, dst, src, shift); + emit(ctx, sra, dst, dst, shift); + } +} + +/* Register move operation (32-bit), optionally with sign extension */ +static void emit_mov_r32(struct jit_context *ctx, const u8 dst[], + const u8 src[], s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, lo(dst), lo(src), off); + break; + default: + /* off == 0 is MOV; the verifier rejects other offsets. */ + emit_mov_r(ctx, lo(dst), lo(src)); + break; + } + clobber_reg(ctx, lo(dst)); + emit_zext_ver(ctx, dst); +} + +/* Register move operation (64-bit), optionally with sign extension */ +static void emit_mov_r64(struct jit_context *ctx, const u8 dst[], + const u8 src[], s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, lo(dst), lo(src), off); + emit(ctx, sra, hi(dst), lo(dst), 31); + break; + case 32: + emit(ctx, move, lo(dst), lo(src)); + emit(ctx, sra, hi(dst), lo(dst), 31); + break; + default: + /* + * off == 0 is ordinary MOV. The verifier rejects other + * offsets; defined exceptions require + * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), + * neither implemented by MIPS. + */ + emit_mov_r(ctx, lo(dst), lo(src)); + emit_mov_r(ctx, hi(dst), hi(src)); + break; + } + clobber_reg64(ctx, dst); +} + /* Load delay slot, if ISA mandates it */ static void emit_load_delay(struct jit_context *ctx) { @@ -512,7 +575,7 @@ static void emit_mul_r64(struct jit_context *ctx, clobber_reg64(ctx, dst); } -/* Helper function for 64-bit modulo */ +/* Helper function for unsigned 64-bit modulo */ static u64 jit_mod64(u64 a, u64 b) { u64 rem; @@ -521,13 +584,22 @@ static u64 jit_mod64(u64 a, u64 b) return rem; } +/* Helper function for signed 64-bit modulo */ +static s64 jit_smod64(s64 a, s64 b) +{ + s64 quot = div64_s64(a, b); + + return a - quot * b; +} + /* ALU div/mod register (64-bit) */ -static void emit_divmod_r64(struct jit_context *ctx, - const u8 dst[], const u8 src[], u8 op) +static void emit_divmod_r64(struct jit_context *ctx, const u8 dst[], + const u8 src[], u8 op, s16 off) { const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ + bool is_signed = off == 1; int exclude, k; u32 addr = 0; @@ -546,11 +618,11 @@ static void emit_divmod_r64(struct jit_context *ctx, switch (BPF_OP(op)) { /* dst = dst / src */ case BPF_DIV: - addr = (u32)&div64_u64; + addr = is_signed ? (u32)&div64_s64 : (u32)&div64_u64; break; /* dst = dst % src */ case BPF_MOD: - addr = (u32)&jit_mod64; + addr = is_signed ? (u32)&jit_smod64 : (u32)&jit_mod64; break; } emit_mov_i(ctx, MIPS_R_T9, addr); @@ -1485,8 +1557,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_mov_i(ctx, hi(dst), 0); } else { - emit_mov_r(ctx, lo(dst), lo(src)); - emit_zext_ver(ctx, dst); + emit_mov_r32(ctx, dst, src, off); } break; /* dst = -dst */ @@ -1516,9 +1587,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T6, imm); - emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); + emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i(ctx, lo(dst), val, alu); } @@ -1546,7 +1617,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_MOD | BPF_X: - emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); + emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = imm (64-bit) */ @@ -1555,8 +1626,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r(ctx, lo(dst), lo(src)); - emit_mov_r(ctx, hi(dst), hi(src)); + emit_mov_r64(ctx, dst, src, off); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: @@ -1599,7 +1669,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) * and then do the operation on this register. */ emit_mov_se_i64(ctx, tmp, imm); - emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); + emit_divmod_r64(ctx, dst, tmp, BPF_OP(code), off); break; /* dst = dst & src (64-bit) */ /* dst = dst | src (64-bit) */ @@ -1629,7 +1699,11 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* dst = dst % src (64-bit) */ case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_divmod_r64(ctx, dst, src, BPF_OP(code)); + emit_divmod_r64(ctx, dst, src, BPF_OP(code), off); + break; + /* dst = bswap(dst) */ + case BPF_ALU64 | BPF_END: + emit_bswap_r64(ctx, dst, imm); break; /* dst = htole(dst) */ /* dst = htobe(dst) */ @@ -1803,6 +1877,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) if (finish_jmp(ctx, jmp, off) < 0) goto toofar; break; + /* PC += imm */ + case BPF_JMP32 | BPF_JA: + if (imm == 0) + break; + if (emit_ja(ctx, imm) < 0) + goto toofar; + break; /* PC += off if dst == src */ /* PC += off if dst != src */ /* PC += off if dst & src */
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 6681cca..994589c 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c
@@ -120,6 +120,58 @@ static void emit_zext_ver(struct jit_context *ctx, u8 dst) emit_zext(ctx, dst); } +/* Sign-extend an 8- or 16-bit field into a native register. */ +static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + /* seb/seh would need a sign-extended 32-bit source first. */ + int shift = 32 - off; + + /* dsll32 and dsra32 add 32 to the shift argument. */ + emit(ctx, dsll32, dst, src, shift); + emit(ctx, dsra32, dst, dst, shift); +} + +/* Register move operation (32-bit), optionally with sign extension */ +static void emit_mov_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, dst, src, off); + break; + default: + /* off == 0 is MOV; the verifier rejects other offsets. */ + emit_mov_r(ctx, dst, src); + break; + } + clobber_reg(ctx, dst); + emit_zext_ver(ctx, dst); +} + +/* Register move operation (64-bit), optionally with sign extension */ +static void emit_mov_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, dst, src, off); + break; + case 32: + emit_sext(ctx, dst, src); + break; + default: + /* + * off == 0 is ordinary MOV. The verifier rejects other + * offsets; defined exceptions require + * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), + * neither implemented by MIPS. + */ + emit_mov_r(ctx, dst, src); + break; + } + clobber_reg(ctx, dst); +} + /* dst = imm (64-bit) */ static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) { @@ -197,8 +249,43 @@ static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) clobber_reg(ctx, dst); } +/* ALU division operation (64-bit) */ +static void emit_div64(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips64r6) { + if (off == 1) + emit(ctx, ddiv_r6, dst, dst, src); + else + emit(ctx, ddivu_r6, dst, dst, src); + } else { + if (off == 1) + emit(ctx, ddiv, dst, src); + else + emit(ctx, ddivu, dst, src); + emit(ctx, mflo, dst); + } +} + +/* ALU modulo operation (64-bit) */ +static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips64r6) { + if (off == 1) + emit(ctx, dmod, dst, dst, src); + else + emit(ctx, dmodu, dst, dst, src); + } else { + if (off == 1) + emit(ctx, ddiv, dst, src); + else + emit(ctx, ddivu, dst, src); + emit(ctx, mfhi, dst); + } +} + /* ALU register operation (64-bit) */ -static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) +static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op, + s16 off) { switch (BPF_OP(op)) { /* dst = dst << src */ @@ -235,25 +322,15 @@ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - if (cpu_has_mips64r6) { - emit(ctx, ddivu_r6, dst, dst, src); - } else { - emit(ctx, ddivu, dst, src); - emit(ctx, mflo, dst); - } + emit_div64(ctx, dst, src, off); break; /* dst = dst % src */ case BPF_MOD: - if (cpu_has_mips64r6) { - emit(ctx, dmodu, dst, dst, src); - } else { - emit(ctx, ddivu, dst, src); - emit(ctx, mfhi, dst); - } + emit_mod64(ctx, dst, src, off); break; default: /* Width-generic operations */ - emit_alu_r(ctx, dst, src, op); + emit_alu_r(ctx, dst, src, op, off); } clobber_reg(ctx, dst); } @@ -655,8 +732,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_zext(ctx, dst); } else { - emit_mov_r(ctx, dst, src); - emit_zext_ver(ctx, dst); + emit_mov_r32(ctx, dst, src, off); } break; /* dst = -dst */ @@ -673,9 +749,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU | BPF_LSH | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i(ctx, dst, val, alu); } @@ -695,10 +771,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_sext(ctx, dst, dst); emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_sext(ctx, dst, dst); emit_alu_i(ctx, dst, val, alu); @@ -713,7 +789,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU | BPF_LSH | BPF_X: - emit_alu_r(ctx, dst, src, BPF_OP(code)); + emit_alu_r(ctx, dst, src, BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = dst >> src */ @@ -732,7 +808,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MOD | BPF_X: emit_sext(ctx, dst, dst); emit_sext(ctx, MIPS_R_T4, src); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = imm (64-bit) */ @@ -741,7 +817,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r(ctx, dst, src); + emit_mov_r64(ctx, dst, src, off); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: @@ -769,9 +845,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i64(ctx, dst, val, alu); } @@ -798,7 +874,11 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_alu_r64(ctx, dst, src, BPF_OP(code)); + emit_alu_r64(ctx, dst, src, BPF_OP(code), off); + break; + /* dst = bswap(dst) */ + case BPF_ALU64 | BPF_END: + emit_bswap_r64(ctx, dst, imm); break; /* dst = htole(dst) */ /* dst = htobe(dst) */ @@ -961,6 +1041,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) if (finish_jmp(ctx, jmp, off) < 0) goto toofar; break; + /* PC += imm */ + case BPF_JMP32 | BPF_JA: + if (imm == 0) + break; + if (emit_ja(ctx, imm) < 0) + goto toofar; + break; /* PC += off if dst == src */ /* PC += off if dst != src */ /* PC += off if dst & src */
diff --git a/arch/openrisc/include/asm/cmpxchg.h b/arch/openrisc/include/asm/cmpxchg.h index 8ee151c..fcf1bc8 100644 --- a/arch/openrisc/include/asm/cmpxchg.h +++ b/arch/openrisc/include/asm/cmpxchg.h
@@ -140,6 +140,8 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, sizeof(*(ptr))); \ }) +#define arch_cmpxchg_local arch_cmpxchg + /* * This function doesn't exist, so you'll get a linker error if * something tries to do an invalidly-sized xchg().
diff --git a/arch/parisc/net/bpf_jit.h b/arch/parisc/net/bpf_jit.h index 8b88969..074115c 100644 --- a/arch/parisc/net/bpf_jit.h +++ b/arch/parisc/net/bpf_jit.h
@@ -467,6 +467,8 @@ static inline u32 hppa_t21_insn(u8 opcode, u8 r2, u8 r1, u8 ext8, u8 t) u64 hppa_div64(u64 div, u64 divisor); u64 hppa_div64_rem(u64 div, u64 divisor); +u64 hppa_sdiv64(u64 div, u64 divisor); +u64 hppa_sdiv64_rem(u64 div, u64 divisor); /* Helper functions that emit HPPA instructions when possible. */
diff --git a/arch/parisc/net/bpf_jit_comp32.c b/arch/parisc/net/bpf_jit_comp32.c index 5ff0cf9..eea0a34 100644 --- a/arch/parisc/net/bpf_jit_comp32.c +++ b/arch/parisc/net/bpf_jit_comp32.c
@@ -321,6 +321,8 @@ static void bpf_put_reg32(const s8 *reg, const s8 *src, extern void $$mulI(void); extern void $$divU(void); extern void $$remU(void); +extern void $$divI(void); +extern void $$remI(void); static void emit_call_millicode(void *func, const s8 arg0, const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) @@ -562,12 +564,14 @@ static void emit_alu_i32(const s8 *dst, s32 imm, } static void emit_alu_r64(const s8 *dst, const s8 *src, - struct hppa_jit_context *ctx, const u8 op) + struct hppa_jit_context *ctx, const u8 op, + bool is_signed) { const s8 *tmp1 = regmap[TMP_REG_1]; const s8 *tmp2 = regmap[TMP_REG_2]; const s8 *rd; const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + void *func; if (op == BPF_MOV) rd = bpf_get_reg64_ref(dst, tmp1, false, ctx); @@ -604,10 +608,12 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, emit_call_libgcc_ll(__muldi3, rd, rs, op, ctx); break; case BPF_DIV: - emit_call_libgcc_ll(&hppa_div64, rd, rs, op, ctx); + func = is_signed ? &hppa_sdiv64 : &hppa_div64; + emit_call_libgcc_ll(func, rd, rs, op, ctx); break; case BPF_MOD: - emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, op, ctx); + func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem; + emit_call_libgcc_ll(func, rd, rs, op, ctx); break; case BPF_LSH: emit_call_libgcc_ll(__ashldi3, rd, rs, op, ctx); @@ -630,7 +636,8 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct hppa_jit_context *ctx, const u8 op) + struct hppa_jit_context *ctx, const u8 op, + bool is_signed) { const s8 *tmp1 = regmap[TMP_REG_1]; const s8 *tmp2 = regmap[TMP_REG_2]; @@ -666,10 +673,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit_call_millicode($$mulI, lo(rd), lo(rs), op, ctx); break; case BPF_DIV: - emit_call_millicode($$divU, lo(rd), lo(rs), op, ctx); + emit_call_millicode(is_signed ? $$divI : $$divU, + lo(rd), lo(rs), op, ctx); break; case BPF_MOD: - emit_call_millicode($$remU, lo(rd), lo(rs), op, ctx); + emit_call_millicode(is_signed ? $$remI : $$remU, + lo(rd), lo(rs), op, ctx); break; case BPF_LSH: emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); @@ -1163,12 +1172,12 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r64(dst, src, ctx, BPF_OP(code)); + emit_alu_r64(dst, src, ctx, BPF_OP(code), off == 1); break; /* dst = -dst */ case BPF_ALU64 | BPF_NEG: - emit_alu_r64(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r64(dst, tmp2, ctx, BPF_OP(code), false); break; case BPF_ALU64 | BPF_MOV | BPF_K: @@ -1211,7 +1220,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + emit_alu_r32(dst, src, ctx, BPF_OP(code), off == 1); break; /* dst = dst OP imm */ @@ -1236,7 +1245,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, BPF_OP(code), false); break; /* dst = BSWAP##imm(dst) */
diff --git a/arch/parisc/net/bpf_jit_comp64.c b/arch/parisc/net/bpf_jit_comp64.c index 54b0d5e..c326fa7 100644 --- a/arch/parisc/net/bpf_jit_comp64.c +++ b/arch/parisc/net/bpf_jit_comp64.c
@@ -502,14 +502,19 @@ static void emit_call(u64 addr, bool fixed, struct hppa_jit_context *ctx) emit_hppa_copy(HPPA_REG_RET0, regmap[BPF_REG_0], ctx); } -static void emit_call_libgcc_ll(void *func, const s8 arg0, - const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) +static void emit_call_libgcc_ll(void *func, const s8 arg0, const s8 arg1, + u8 opcode, bool is_signed, struct hppa_jit_context *ctx) { u64 func_addr; if (BPF_CLASS(opcode) == BPF_ALU) { - emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); - emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + if (is_signed) { + emit_hppa64_sext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_sext32(arg1, HPPA_REG_ARG1, ctx); + } else { + emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + } } else { emit_hppa_copy(arg0, HPPA_REG_ARG0, ctx); emit_hppa_copy(arg1, HPPA_REG_ARG1, ctx); @@ -600,6 +605,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, u8 rd = -1, rs = -1, code = insn->code; s16 off = insn->off; s32 imm = insn->imm; + bool is_signed; + void *func; init_regs(&rd, &rs, insn, ctx); @@ -656,29 +663,39 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, fallthrough; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: - emit_call_libgcc_ll(__muldi3, rd, rs, code, ctx); + emit_call_libgcc_ll(__muldi3, rd, rs, code, false, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: - emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + is_signed = (off == 1); + emit_imm(HPPA_REG_T1, + is64 || is_signed ? (s64)(s32)imm : (u32)imm, + HPPA_REG_T2, ctx); rs = HPPA_REG_T1; fallthrough; case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: - emit_call_libgcc_ll(&hppa_div64, rd, rs, code, ctx); + is_signed = (off == 1); + func = is_signed ? &hppa_sdiv64 : &hppa_div64; + emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: - emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + is_signed = (off == 1); + emit_imm(HPPA_REG_T1, + is64 || is_signed ? (s64)(s32)imm : (u32)imm, + HPPA_REG_T2, ctx); rs = HPPA_REG_T1; fallthrough; case BPF_ALU | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, code, ctx); + is_signed = (off == 1); + func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem; + emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break;
diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index 1727701..a2f9320 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c
@@ -190,3 +190,17 @@ u64 hppa_div64_rem(u64 div, u64 divisor) div64_u64_rem(div, divisor, &rem); return rem; } + +u64 hppa_sdiv64(u64 div, u64 divisor) +{ + s64 sdiv = div64_s64((s64)div, (s64)divisor); + + return (u64)sdiv; +} + +u64 hppa_sdiv64_rem(u64 div, u64 divisor) +{ + s64 sdiv = div64_s64((s64)div, (s64)divisor); + + return (u64)((s64)div - sdiv * (s64)divisor); +}
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 1eb8e34..e0b7901 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig
@@ -442,7 +442,6 @@ CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_USER_COPY=m -CONFIG_TEST_BPF=m CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 2d139c8..708a32f 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h
@@ -29,6 +29,10 @@ #define __KVM_HAVE_ARCH_VCPU_DEBUGFS +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && !defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) +#define __KVM_NO_MMIO +#endif + #define KVM_MAX_VCPUS NR_CPUS #define KVM_MAX_VCORES NR_CPUS @@ -49,6 +53,9 @@ #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#ifdef CONFIG_KVM_MPIC +#define __KVM_HAVE_MSI +#endif #define KVM_HALT_POLL_NS_DEFAULT 10000 /* 10 us */
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index bce5469..55bcf03 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c
@@ -122,6 +122,7 @@ void inc_irq_stat_and_enable(enum irq_stat_counts which) set_bit(which, irq_stat_count_show); } +#ifdef CONFIG_PROC_FS int arch_show_interrupts(struct seq_file *p, int prec) { const struct irq_stat_info *info = irq_stat_info; @@ -155,6 +156,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) #endif return 0; } +#endif /* CONFIG_PROC_FS */ static int __init irq_init_stats(void) {
diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index b6bc2fc..cacc22e 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig
@@ -30,7 +30,6 @@ config KVM_BOOK3S_32_HANDLER bool select KVM_BOOK3S_HANDLER - select KVM_MMIO config KVM_BOOK3S_64_HANDLER bool @@ -38,7 +37,6 @@ config KVM_BOOK3S_PR_POSSIBLE bool - select KVM_MMIO config KVM_BOOK3S_HV_POSSIBLE bool @@ -201,7 +199,6 @@ depends on PPC_E500 && !PPC_E500MC depends on !CONTEXT_TRACKING_USER select KVM - select KVM_MMIO help Support running unmodified E500 guest kernels in virtual machines on E500v2 host processors. @@ -216,7 +213,6 @@ depends on PPC_E500MC depends on !CONTEXT_TRACKING_USER select KVM - select KVM_MMIO select KVM_BOOKE_HV help Support running unmodified E500MC/E5500/E6500 guest kernels in @@ -232,7 +228,6 @@ depends on KVM && PPC_E500 select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI help Enable support for emulating MPIC devices inside the host kernel, rather than relying on userspace to emulate.
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 140eadb..22283d4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c
@@ -269,7 +269,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) * stolen time for a vcore when it is inactive, or for a vcpu * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of * a misnomer; it means that the vcpu task is not executing in - * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in + * the KVM_RUN ioctl, i.e. it is in userspace or elsewhere in * the kernel. We don't have any way of dividing up that time * between time that the vcpu is genuinely stopped, time that * the task is actively working on behalf of the vcpu, and time
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index cfe01c3..e0b812d 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -267,8 +267,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); - cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); - cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + if (bpf_prog_uses_arena_insns(fp)) { + cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); + cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + } cgctx.is_subprog = bpf_is_subprog(fp); cgctx.exception_boundary = fp->aux->exception_boundary; cgctx.exception_cb = fp->aux->exception_cb;
diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index a306005..e28f058 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h
@@ -47,6 +47,8 @@ #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6) #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM #define KVM_HAVE_MMU_RWLOCK
diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index ec2cee0..2e3c4e6 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig
@@ -22,13 +22,10 @@ depends on RISCV_SBI && MMU select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI - select HAVE_KVM_READONLY_MEM select HAVE_KVM_DIRTY_RING_ACQ_REL select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING - select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 857f9a5..b50de11 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c
@@ -186,7 +186,7 @@ void __init arch_mm_preinit(void) */ unsigned long size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); - swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); + swiotlb_adjust_size(min(swiotlb_default_pool_size(), size)); swiotlb = true; swiotlb_flags |= SWIOTLB_ANY; }
diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 419b9d7..5c4f530 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h
@@ -12,6 +12,9 @@ #include <linux/bpf.h> #include <linux/filter.h> +#define RV_MAX_REG_ARGS 8 +#define RV_EXTRA_STK_ARGS (RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS) + /* verify runtime detection extension status */ #define rv_ext_enabled(ext) \ (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) @@ -82,6 +85,7 @@ struct rv_jit_context { unsigned long flags; int stack_size; int tcc_offset; + int stack_arg_sz; u64 arena_vm_start; u64 user_vm_start; };
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 01fe667..788fcc2 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -15,7 +15,6 @@ #include <asm/percpu.h> #include "bpf_jit.h" -#define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) /* imm that allows emit_imm to emit max count insns */ @@ -23,11 +22,11 @@ /* fentry and TCC init insns will be skipped on tailcall */ #define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) -#define RV_REG_TCC RV_REG_A6 +#define RV_REG_TCC RV_REG_T5 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { - [BPF_REG_0] = RV_REG_A5, + [BPF_REG_0] = RV_REG_T6, [BPF_REG_1] = RV_REG_A0, [BPF_REG_2] = RV_REG_A1, [BPF_REG_3] = RV_REG_A2, @@ -47,13 +46,13 @@ static const int pt_regmap[] = { [RV_REG_A2] = offsetof(struct pt_regs, a2), [RV_REG_A3] = offsetof(struct pt_regs, a3), [RV_REG_A4] = offsetof(struct pt_regs, a4), - [RV_REG_A5] = offsetof(struct pt_regs, a5), [RV_REG_S1] = offsetof(struct pt_regs, s1), [RV_REG_S2] = offsetof(struct pt_regs, s2), [RV_REG_S3] = offsetof(struct pt_regs, s3), [RV_REG_S4] = offsetof(struct pt_regs, s4), [RV_REG_S5] = offsetof(struct pt_regs, s5), [RV_REG_T0] = offsetof(struct pt_regs, t0), + [RV_REG_T6] = offsetof(struct pt_regs, t6), }; enum { @@ -239,7 +238,7 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) - emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); + emit_addiw(RV_REG_A0, regmap[BPF_REG_0], 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } @@ -498,6 +497,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx) +{ + int idx = off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(rd, RV_REG_A5 + idx, ctx); + return; + } + + emit_ldx_insn(rd, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_FP, BPF_DW, false, ctx); +} + static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) { emit_imm(RV_REG_T1, imm, ctx); @@ -515,6 +526,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_imm(RV_REG_A5 + idx, imm, ctx); + return; + } + + emit_imm(RV_REG_T1, imm, ctx); + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_T1, BPF_DW, ctx); +} + static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) { if (is_12b_int(off)) { @@ -531,6 +555,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(RV_REG_A5 + idx, rs, ctx); + return; + } + + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, rs, BPF_DW, ctx); +} + static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, struct rv_jit_context *ctx) { @@ -747,6 +783,8 @@ static int add_exception_handler(const struct bpf_insn *insn, int dst_reg, if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + !(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX && + BPF_CLASS(insn->code) == BPF_LDX) && BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return 0; @@ -875,7 +913,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return ret; } -static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx) +static void store_args(int nr_arg_slots, int args_off, int stack_args_off, + struct rv_jit_context *ctx) { int i; @@ -883,8 +922,8 @@ static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ct if (i < RV_MAX_REG_ARGS) { emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); } else { - /* skip slots for T0 and FP of traced function */ - emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); + emit_ld(RV_REG_T1, stack_args_off + + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); } args_off -= 8; @@ -1100,7 +1139,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); if (save_ret) - stack_size += 16; /* Save both A5 (BPF R0) and A0 */ + stack_size += 16; /* Save both RV_REG_T6 (BPF R0) and RV_REG_A0 */ retval_off = stack_size; stack_size += nr_arg_slots * 8; @@ -1179,7 +1218,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, func_meta = nr_arg_slots; emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); - store_args(nr_arg_slots, args_off, ctx); + /* + * A direct struct_ops call has its first stack argument at the incoming + * SP, which the trampoline keeps as FP. The fentry path pushes the + * parent frame first, so its incoming stack arguments start at FP + 16. + */ + store_args(nr_arg_slots, args_off, is_struct_ops ? 0 : 16, ctx); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ @@ -1675,6 +1719,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zextw(rd, rd, ctx); break; + /* JUMP reg */ + case BPF_JMP | BPF_JA | BPF_X: + emit_jalr(RV_REG_ZERO, rd, 0, ctx); + break; + /* JUMP off */ case BPF_JMP | BPF_JA: case BPF_JMP32 | BPF_JA: @@ -1844,11 +1893,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return -EINVAL; for (idx = 0; idx < fm->nr_args; idx++) { - u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; + u8 arg_sz = fm->arg_size[idx]; - if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) - return -EINVAL; + if (arg_sz == 8 || (arg_sz != 4 && !sign)) + continue; + + if (idx < RV_MAX_REG_ARGS) { + if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx)) + return -EINVAL; + } else { + emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx); + if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx)) + return -EINVAL; + emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx); + } } } @@ -1873,6 +1932,13 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; } + + /* indirect call to a bpf subprog via the address in rd */ + case BPF_JMP | BPF_CALL | BPF_X: + mark_call(ctx); + emit_jalr(RV_REG_RA, rd, 0, ctx); + break; + /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(i, ctx)) @@ -1930,18 +1996,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + /* LDX | PROBE_MEM32SX: dst = *(signed size *)(src + RV_REG_ARENA + off) */ + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W: { bool sign_ext; sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || - BPF_MODE(insn->code) == BPF_PROBE_MEMSX; + BPF_MODE(insn->code) == BPF_PROBE_MEMSX || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) { emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); rs = RV_REG_T2; } - emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); + if (is_stack_arg_ldx(insn)) + emit_stack_arg_ldx(rd, off, ctx); + else + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); ret = add_exception_handler(insn, rd, ctx); if (ret) @@ -1971,7 +2046,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T3; } - emit_st(rd, off, imm, BPF_SIZE(code), ctx); + if (is_stack_arg_st(insn)) + emit_stack_arg_st(off, imm, ctx); + else + emit_st(rd, off, imm, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -1993,7 +2071,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T2; } - emit_stx(rd, off, rs, BPF_SIZE(code), ctx); + if (is_stack_arg_stx(insn)) + emit_stack_arg_stx(off, rs, ctx); + else + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -2043,9 +2124,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { - int i, stack_adjust = 0, store_offset, bpf_stack_adjust; + int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz; - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); if (bpf_stack_adjust) mark_fp(ctx); @@ -2143,6 +2224,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; @@ -2170,10 +2256,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) if (insn->imm == BPF_CMPXCHG) return rv_ext_enabled(ZACAS); break; - case BPF_LDX | BPF_MEMSX | BPF_B: - case BPF_LDX | BPF_MEMSX | BPF_H: - case BPF_LDX | BPF_MEMSX | BPF_W: - return false; } } @@ -2211,3 +2293,13 @@ bool bpf_jit_supports_timed_may_goto(void) { return true; } + +bool bpf_jit_supports_stack_args(void) +{ + return true; +} + +bool bpf_jit_supports_callx(void) +{ + return true; +}
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index cbfcd28..f2497ec 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c
@@ -26,10 +26,13 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) int ret; ret = bpf_jit_emit_insn(insn, ctx, extra_pass); - if (ret > 0) - i++; /* skip the next instruction */ if (offset) offset[i] = ctx->ninsns; + if (ret > 0) { + i++; /* skip the next instruction */ + if (offset) + offset[i] = ctx->ninsns; + } if (ret < 0) return ret; } @@ -48,6 +51,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; + u16 stack_arg_cnt; if (!prog->jit_requested) return prog; @@ -69,8 +73,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr goto skip_init_ctx; } - ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } + + stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); + /* First 3 stack args in regs, rest on stack */ + ctx->stack_arg_sz = stack_arg_cnt > RV_EXTRA_STK_ARGS ? + round_up((stack_arg_cnt - RV_EXTRA_STK_ARGS) * 8, STACK_ALIGN) : 0; + ctx->prog = prog; ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) @@ -169,6 +181,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); + + /* + * bpf_prog_update_insn_ptrs() wants the start of each insn, so + * shift ctx->offset[] by one and get insn 0 from the prologue. + */ + for (i = prog->len - 1; i > 0; i--) + ctx->offset[i] = ctx->offset[i - 1]; + ctx->offset[0] = ninsns_rvoff(ctx->prologue_len); + bpf_prog_update_insn_ptrs(prog, ctx->offset, jit_data->ro_image); out_offset: kvfree(ctx->offset); kfree(jit_data); @@ -237,6 +258,7 @@ void bpf_jit_free(struct bpf_prog *prog) kvfree(jit_data->ctx.offset); kfree(jit_data); } + prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); hdr = bpf_jit_binary_pack_hdr(prog); bpf_jit_binary_pack_free(hdr, NULL); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 02c637d..fcf850f2 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S
@@ -10,7 +10,7 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. */ @@ -20,8 +20,8 @@ REG_S s0, 6*SZREG(sp) addi s0, sp, 8*SZREG - /* Save BPF registers R0-R5 (a5, a0-a4) */ - REG_S a5, 5*SZREG(sp) + /* Save BPF registers R0-R5 (t6, a0-a4) */ + REG_S t6, 5*SZREG(sp) REG_S a0, 4*SZREG(sp) REG_S a1, 3*SZREG(sp) REG_S a2, 2*SZREG(sp) @@ -38,7 +38,7 @@ REG_L a2, 2*SZREG(sp) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) - REG_L a5, 5*SZREG(sp) + REG_L t6, 5*SZREG(sp) REG_L s0, 6*SZREG(sp) REG_L ra, 7*SZREG(sp)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a8b891cd..2c46cfc 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig
@@ -181,7 +181,7 @@ select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_SMP_IDLE_THREAD - select GENERIC_IOREMAP if PCI + select GENERIC_IOREMAP select GLOB select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL @@ -766,9 +766,6 @@ endif # PCI -config HAS_IOMEM - def_bool PCI - config CHSC_SCH def_tristate m prompt "Support for CHSC subchannels"
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 3892d9d..ac30bf5 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig
@@ -941,6 +941,5 @@ CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y CONFIG_TEST_BITOPS=m -CONFIG_TEST_BPF=m CONFIG_PAGE_TABLE_CHECK=y CONFIG_PAGE_TABLE_CHECK_ENFORCED=y
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 95720e8..a02d8bf 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig
@@ -865,4 +865,3 @@ CONFIG_KPROBES_SANITY_TEST=m CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y -CONFIG_TEST_BPF=m
diff --git a/arch/s390/include/arm64/kvm_emulate.h b/arch/s390/include/arm64/kvm_emulate.h new file mode 100644 index 0000000..416c4f9 --- /dev/null +++ b/arch/s390/include/arm64/kvm_emulate.h
@@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_KVM_EMULATE_H__ +#define __S390_ARM64_KVM_EMULATE_H__ + +#include <asm/fault.h> +#include <linux/kvm_host.h> + +#include <arm64/kvm_nested.h> +#include <arm64/ptrace.h> +#include <arm64/kvm_arm.h> +#include <arm64/sysreg.h> + +static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.pc; +} + +static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.pstate; +} + +static __always_inline unsigned long *vcpu_sp_el0(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.sp_el0; +} + +static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.sp_el1; +} + +static __always_inline __vector128 *vcpu_vreg(struct kvm_vcpu *vcpu, int off) +{ + return &vcpu->arch.ctxt.vregs[off]; +} + +static __always_inline u32 *vcpu_fpsr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.fpsr; +} + +static __always_inline u32 *vcpu_fpcr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.fpcr; +} + +static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) +{ + return false; +} + +static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.esr_el2; +} + +static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.hcr_el2; +} + +static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.far_el2; +} + +static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) +{ + return gfn_to_gpa(vcpu->arch.sae_block.hai.teid.addr); +} + +static inline u16 kvm_vcpu_fault_pic(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.pic & PGM_INT_CODE_MASK; +} + +static __always_inline +bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_fault_pic(vcpu) == PGM_PROTECTION; +} + +static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu) +{ + return true; +} + +static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) +{ + return false; +} + +static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) +{ + return false; +} + +static inline int kvm_vcpu_abt_gltl(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.gltl; +} + +static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu) +{ + u32 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK; + + return mode != PSR_MODE_EL0t; +} + +static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) +{ + *vcpu_pc(vcpu) += 4; + *vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK; + + /* advance the singlestep state machine */ + *vcpu_cpsr(vcpu) &= ~SPSR_ELx_SS; +} + +static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) +{ + memset(vcpu->arch.ctxt.vregs, 0, sizeof(vcpu->arch.ctxt.vregs)); + vcpu->arch.sae_block.fpsr = 0; + vcpu->arch.sae_block.fpcr = 0; +} + +#include <arm64/kvm_emulate-gen.h> + +#endif /* __S390_ARM64_KVM_EMULATE_H__ */
diff --git a/arch/s390/include/arm64/kvm_nested.h b/arch/s390/include/arm64/kvm_nested.h new file mode 100644 index 0000000..e950b1a --- /dev/null +++ b/arch/s390/include/arm64/kvm_nested.h
@@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef ASM_KVM_NESTED_H +#define ASM_KVM_NESTED_H + +static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) +{ + return false; +} + +#endif /* ASM_KVM_NESTED_H */
diff --git a/arch/s390/include/arm64/ptrace.h b/arch/s390/include/arm64/ptrace.h new file mode 100644 index 0000000..449b8558 --- /dev/null +++ b/arch/s390/include/arm64/ptrace.h
@@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_PTRACE_H__ +#define __S390_ARM64_PTRACE_H__ + +#include <arm64/sysreg.h> +#include <uapi/arm64/ptrace.h> + +#include <arm64/ptrace-gen.h> + +#endif /* __S390_ARM64_PTRACE_H__ */
diff --git a/arch/s390/include/arm64/sysreg.h b/arch/s390/include/arm64/sysreg.h new file mode 100644 index 0000000..ac6b638 --- /dev/null +++ b/arch/s390/include/arm64/sysreg.h
@@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_SYSREG_H__ +#define __S390_ARM64_SYSREG_H__ + +#include <linux/bits.h> +#include <linux/stringify.h> +#include <linux/kasan-tags.h> +#include <linux/kconfig.h> + +#include <arm64/sysreg-gen.h> + +#endif /* __S390_ARM64_SYSREG_H__ */
diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index d4da443..faa2afb 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h
@@ -6,6 +6,7 @@ #include <asm/bug.h> #include <asm/fpu.h> #include <asm/nospec-branch.h> +#include <asm/sae.h> #include <asm-generic/asm-prototypes.h> #endif /* _ASM_S390_PROTOTYPES_H */
diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index bb63fa4..ad3108e 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h
@@ -123,6 +123,7 @@ enum { HWCAP_NR_NNPA = 20, HWCAP_NR_PCI_MIO = 21, HWCAP_NR_SIE = 22, + HWCAP_NR_SAE = 23, HWCAP_NR_MAX }; @@ -150,6 +151,7 @@ enum { #define HWCAP_NNPA BIT(HWCAP_NR_NNPA) #define HWCAP_PCI_MIO BIT(HWCAP_NR_PCI_MIO) #define HWCAP_SIE BIT(HWCAP_NR_SIE) +#define HWCAP_SAE BIT(HWCAP_NR_SAE) /* * These are used to set parameters in the core dumps.
diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h index faddb9a..ef0490e 100644 --- a/arch/s390/include/asm/io.h +++ b/arch/s390/include/asm/io.h
@@ -27,14 +27,8 @@ void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr); /* * I/O memory mapping functions. */ -#define ioremap_prot ioremap_prot -#define iounmap iounmap - #define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL) -#define ioremap_wc(addr, size) \ - ioremap_prot((addr), (size), pgprot_writecombine(PAGE_KERNEL)) - static inline void __iomem *ioport_map(unsigned long port, unsigned int nr) { return NULL; @@ -46,6 +40,12 @@ static inline void ioport_unmap(void __iomem *p) #ifdef CONFIG_PCI +#define ioremap_prot ioremap_prot +#define iounmap iounmap + +#define ioremap_wc(addr, size) \ + ioremap_prot((addr), (size), pgprot_writecombine(PAGE_KERNEL)) + /* * s390 needs a private implementation of pci_iomap since ioremap with its * offset parameter isn't sufficient. That's because BAR spaces are not
diff --git a/arch/s390/include/asm/kvm.h b/arch/s390/include/asm/kvm.h new file mode 100644 index 0000000..aeb11dc --- /dev/null +++ b/arch/s390/include/asm/kvm.h
@@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef KVM_S390_ARM64 +#include <uapi/arm64/kvm.h> +#else +#include <uapi/asm/kvm.h> +#endif
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 1473259..ac3440a 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h
@@ -3,7 +3,11 @@ #ifndef ASM_KVM_HOST_H #define ASM_KVM_HOST_H +#ifdef KVM_S390_ARM64 +#include <asm/kvm_host_arm64.h> +#else #include <asm/kvm_host_s390.h> +#endif #define PGM_OPERATION 0x01 #define PGM_PRIVILEGED_OP 0x02
diff --git a/arch/s390/include/asm/kvm_host_arm64.h b/arch/s390/include/asm/kvm_host_arm64.h new file mode 100644 index 0000000..82343e7 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_arm64.h
@@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_KVM_HOST_ARM64_H +#define ASM_KVM_HOST_ARM64_H + +#include <linux/bug.h> + +#include <asm/kvm_host_types.h> +#include <asm/debug.h> + +#define vcpu_gp_regs(v) ((v)->arch.sae_block.gpr) + +#include <arm64/kvm_host.h> +#include <arm64/ptrace.h> + +#include <asm/sae.h> + +#define KVM_HAVE_MMU_RWLOCK +#define KVM_MAX_VCPUS 1 +#define __KVM_HAVE_VCPU_RUN_PID_CHANGE + +#define KVM_S390_ARM64_IMPL_FEATURES 0 +#define KVM_HALT_POLL_NS_DEFAULT 50000 + +#define KVM_S390_MANAGES_S390_GUEST 0 + +/* Minimal (=no) vgic definitions */ +#define KVM_IRQCHIP_NUM_PINS 1 +#define irqchip_in_kernel(_k) false + +#define __ctxt_sys_reg(ctx, reg) NULL +struct kvm_cpu_context { + /* + * These are just for 32 bit, which we don't have, making them RES0. + * They are exposed to user space. + */ + u64 spsr_abt; + u64 spsr_und; + u64 spsr_irq; + u64 spsr_fiq; + + __vector128 __aligned(16) vregs[32]; +}; + +struct kvm_vcpu_arch { + struct kvm_sae_block sae_block; + struct kvm_sae_save_area save_area; + struct kvm_cpu_context ctxt; + + u32 host_acrs[NUM_ACRS]; + + /* Hypervisor Configuration Register */ + u64 hcr_el2; + + /* Configuration flags, set once and for all before the vcpu can run */ + u8 cflags; + + /* Input flags to the hypervisor code, potentially cleared after use */ + u8 iflags; + + /* State flags for kernel bookkeeping, unused by the hypervisor code */ + u8 sflags; + + /* + * Don't run the guest (internal implementation need). + * + * Contrary to the flags above, this is set/cleared outside of + * a vcpu context, and thus cannot be mixed with the flags + * themselves (or the flag accesses need to be made atomic). + */ + bool pause; + + /* vcpu power state */ + struct kvm_mp_state mp_state; + /* lock for mp_state & reset_state.reset */ + spinlock_t mp_state_lock; + + /* vcpu reset state */ + struct vcpu_reset_state reset_state; + + struct kvm_s390_mmu_cache *mc; + + void *debugfs_state_data; +}; + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + /* ARM64 stats */ + u64 hvc_exit_stat; + u64 wfe_exit_stat; + u64 wfi_exit_stat; + u64 mmio_exit_user; + u64 mmio_exit_kernel; + u64 signal_exits; + u64 exits; + /* GMAP stats */ + u64 pfault_sync; +}; + +#define kvm_has_mte(_kvm) false +#define vcpu_has_sve(_vcpu) false +#define vcpu_has_ptrauth(_vcpu) false + +struct kvm_arch_memory_slot { +}; + +struct kvm_arch { + struct gmap *gmap; + u64 guest_phys_size; + + /* VM-wide vCPU feature set */ + unsigned long flags; + + /* Protects VM-scoped configuration data */ + struct mutex config_lock; + + debug_info_t *dbf; + + DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES); + + unsigned long mem_limit; +}; + +static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) +{ + return test_bit(feature, ka->vcpu_features); +} + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; +}; + +static inline bool system_has_full_ptr_auth(void) +{ + return true; +} + +#define kvm_vm_is_protected(_kvm) false +#define vcpu_is_protected(_vcpu) false + +#define vcpu_is_loaded(_vcpu) ((_vcpu)->cpu != -1) + +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) + +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +u32 get_kvm_ipa_limit(void); + +/* unused, but required functions */ +static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_sync_events(struct kvm *kvm) {} +static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, + unsigned long mask) {} + +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu); +void kvm_reset_vcpu(struct kvm_vcpu *vcpu); + +/* arm64 guests do not use async-pf. Defined because Kbuild requires it as s390 kvm turns it on. */ +#define ASYNC_PF_PER_VCPU 0 +struct kvm_arch_async_pf { + unsigned long pfault_token; +}; + +#define __unsupp_async_call(fn) WARN_ONCE(true, "async not supported on kvm-arm64 %s", fn) + +static inline bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) +{ + __unsupp_async_call(__func__); + return false; +}; + +static inline void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); +}; + +static inline bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); + return false; +}; + +static inline void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); +}; + +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) +{ + __unsupp_async_call(__func__); +}; + +#define kvm_supports_32bit_el0() false + +#endif /* ASM_KVM_HOST_ARM64_H */
diff --git a/arch/s390/include/asm/kvm_host_arm64_types.h b/arch/s390/include/asm/kvm_host_arm64_types.h new file mode 100644 index 0000000..8783a66 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_arm64_types.h
@@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_KVM_HOST_ARM64_TYPES_H +#define ASM_KVM_HOST_ARM64_TYPES_H + +#include <linux/types.h> +#include <linux/kvm_types.h> +#include <linux/compiler_attributes.h> +#include <asm/page.h> +#include <asm/fault.h> + +struct kvm_sae_block { + u64 _0000[16]; /* 0x0000 */ +#define SAE_ICPTR_SPURIOUS 0x00 +#define SAE_ICPTR_VALIDITY 0x01 +#define SAE_ICPTR_HOST_ACCESS_EXCEPTION 0x02 +#define SAE_ICPTR_SYNCHRONOUS_EXCEPTION 0x03 +#define SAE_ICPTR_TIMER 0x04 +#define SAE_ICPTR_PE_INTERCOMM 0x05 +#define SAE_ICPTR_GUEST_ADDRESS_SIZE 0x06 +#define SAE_ICPTR_STOP 0x07 +#define SAE_ICPTR_MIO_ADDRESS 0x08 +#define SAE_ICPTR_PMU 0x09 +#define SAE_ICPTR_MAINTENANCE 0x0a + u8 icptr; /* 0x0080 */ + u8 _0081[7]; /* 0x0081 */ + u64 scad; /* 0x0088 */ + u64 _0090[16]; /* 0x0090 */ + u32 cntp_ctl; /* 0x0110 */ + u32 cntv_ctl; /* 0x0114 */ + u8 irq_ctl; /* 0x0118 */ + u8 _0119[7]; /* 0x0119 */ + struct { + u64 ich_hcr_el2; /* 0x0120 */ + u64 ich_vmcr_el2; /* 0x0128 */ + u64 ich_ap0r0_el2; /* 0x0130 */ + u64 ich_ap1r0_el2; /* 0x0138 */ + u64 _0140[2]; /* 0x0140 */ + u64 ich_lrn_el2[4]; /* 0x0150 */ + u64 _0170[4]; /* 0x0170 */ + } ic_regs; + u64 _0190[12]; /* 0x0190 */ + u64 contextidr_el1; /* 0x01f0 */ + u32 wip; /* 0x01f8 */ + u32 _01fc; /* 0x01fc */ +#define SAE_SD_FORMAT_0 0x00 + u8 sdf; /* 0x0200 */ + u8 _0201[7]; /* 0x0201 */ + u64 mso; /* 0x0208 */ + u64 msl; /* 0x0210 */ + u64 hbasce; /* 0x0218 */ + u64 _0220; /* 0x0220 */ + u64 gpto; /* 0x0228 */ + u64 ic; /* 0x0230 */ + u64 ec; /* 0x0238 */ + u64 save_area; /* 0x0240 */ + u64 _0248[7]; /* 0x0248 */ + u8 _0280[6]; /* 0x0280 */ + u16 lrcpua; /* 0x0286 */ + u64 pstate; /* 0x0288 */ + u64 pc; /* 0x0290 */ + u64 sp_el0; /* 0x0298 */ + u64 sp_el1; /* 0x02a0 */ + u64 _02a8; /* 0x02a8 */ + struct { /* 0x02b0 */ + u32 _fpcr_res0; + u32 fpcr; + }; + struct { /* 0x02b8 */ + u32 _fpsr_res0; + u32 fpsr; + }; + u16 sve_pregs[16]; /* 0x02c0 */ + u16 sve_ffr; /* 0x02e0 */ + u8 _02e2[6]; /* 0x02e2 */ + u64 _02e8[3]; /* 0x02e8 */ + + u64 gpr[31]; /* 0x0300 */ + u64 _03f8; /* 0x03f8 */ + + union { + u64 icptd[8]; /* 0x0400 */ + /* validity-interception reason; icptr 0x01 */ + u16 vir; /* 0x0400 */ + /* host access interception details; icptr 0x02 */ + struct { + u64 esr_el2; /* 0x0400 */ + u8 _0408[6]; /* 0x0408 */ + u16 pic; /* 0x040e */ + union teid teid; /* 0x0410 */ + gva_t far_el2; /* 0x0418 */ + gva_t vaddr; /* 0x0420 */ + u64 suppl; /* 0x0428 */ + u8 gltl; /* 0x0430 */ + u8 _0431[7]; /* 0x0431 */ + u64 _0438; /* 0x0438 */ + } hai; + /* exception-interception details; icptr 0x03 */ + struct { + u64 esr_el2; /* 0x0400 */ + u64 _0408[2]; /* 0x0408 */ + gva_t far_el2; /* 0x0418 */ + } trap; + /* timer-interception reason; icptr 0x04 */ +#define SAE_IR_TIMER_ID_VIRT BIT(6) +#define SAE_IR_TIMER_ID_PHYS BIT(7) + u8 tir; /* 0x0400 */ + }; + u64 _0440[376]; /* 0x0440 */ +} __packed __aligned(PAGE_SIZE); +static_assert(sizeof(struct kvm_sae_block) == PAGE_SIZE); + +struct kvm_sae_save_area { +#define SAE_SAVE_AREA_FORMAT_0 0x00 + u8 saf; /* 0x0000 */ + u8 _0001[5]; /* 0x0001 */ +#define SAE_SAS_VALID BIT(0) + u16 sas; /* 0x0006 */ + u64 sdo; /* 0x0008 */ + u64 _0010[2]; /* 0x0010 */ + u64 regs[508]; /* 0x0020 */ +} __packed __aligned(PAGE_SIZE); +static_assert(sizeof(struct kvm_sae_save_area) == PAGE_SIZE); + +#endif /* ASM_KVM_HOST_ARM64_TYPES_H */
diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index cd692f8..f780260 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h
@@ -27,13 +27,20 @@ #include <asm/isc.h> #include <asm/guarded_storage.h> +#define kvm_file_to_kvm_arch s390 + #define KVM_HAVE_MMU_RWLOCK #define KVM_MAX_VCPUS 255 #define KVM_INTERNAL_MEM_SLOTS 1 +#define __KVM_HAVE_INVALID_WAKEUPS +#define __KVM_HAVE_NO_POLL + #define KVM_S390_MANAGES_S390_GUEST 1 +#define __KVM_NO_MMIO + /* * These seem to be used for allocating ->chip in the routing table, which we * don't use. 1 is as small as we can get to reduce the needed memory. If we @@ -720,7 +727,7 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} void kvm_arch_free_vm(struct kvm *kvm); struct zpci_kvm_hook { - int (*kvm_register)(void *opaque, struct kvm *kvm); + int (*kvm_register)(void *opaque, struct file *kvm_file); void (*kvm_unregister)(void *opaque); };
diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index e5bdba0..d308739 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h
@@ -3,6 +3,10 @@ #ifndef ASM_KVM_HOST_TYPES_H #define ASM_KVM_HOST_TYPES_H +#ifdef KVM_S390_ARM64 +#include <asm/kvm_host_arm64_types.h> +#else #include <asm/kvm_host_s390_types.h> +#endif /* KVM_S390_ARM64 */ #endif /* ASM_KVM_HOST_TYPES_H */
diff --git a/arch/s390/include/asm/sae.h b/arch/s390/include/asm/sae.h new file mode 100644 index 0000000..26d89fa --- /dev/null +++ b/arch/s390/include/asm/sae.h
@@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_S390_SAE_H +#define __ASM_S390_SAE_H + +#include <linux/linkage.h> +#include <linux/types.h> + +#ifndef __ASSEMBLER__ + +/* defined in arch/s390/kernel/entry.S */ +asmlinkage void sae(phys_addr_t sae_block_phys); + +#include <linux/io.h> +#include <asm/kvm_host_arm64_types.h> + +/** + * stiasrm() - STore and Invalidate Arm System Register Multiple + * @save_area: Pointer to SAE save area + * + * Store the guest system registers to the save area. + * The values in the guest are no longer valid. + */ +static __always_inline void stiasrm(struct kvm_sae_save_area *save_area) +{ + asm_inline volatile( + " .insn rre,0xb9a70000,%[r1],0\n" + : "+m" (*save_area) + : [r1] "a" (save_area) + ); +} + +/** + * lasrm() - Load Arm System Register Multiple + * + * @save_area: Pointer to SAE save area + * + * Load the system registers from save_area into the guest. + */ +static __always_inline void lasrm(struct kvm_sae_save_area *save_area) +{ + asm_inline volatile( + " .insn rre,0xb9a60000,%[r1],0\n" + : + : "m" (*save_area), [r1] "a" (save_area) + ); +} + +#endif /* !__ASSEMBLER__ */ +#endif /* __ASM_S390_SAE_H */
diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 4406655..68f61b7 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h
@@ -56,7 +56,9 @@ struct sclp_core_entry { u8 siif : 1; u8 sigpif : 1; u8 : 3; - u8 reserved2[3]; + u8 aef: 1; + u8 : 7; + u8 reserved2[2]; u8 : 2; u8 ib : 1; u8 cei : 1; @@ -109,6 +111,7 @@ struct sclp_info { unsigned char has_aeni : 1; unsigned char has_aisi : 1; unsigned char has_astfleie2 : 1; + unsigned char has_aef : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp;
diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index ac3606c..2d332d7 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h
@@ -59,6 +59,7 @@ static inline bool on_stack(struct stack_info *info, struct stack_frame { union { unsigned long empty[9]; + /* SIE stack frame */ struct { unsigned long sie_control_block; unsigned long sie_savearea; @@ -68,6 +69,10 @@ struct stack_frame { unsigned long sie_guest_asce; unsigned long sie_irq; }; + /* SAE stack frame */ + struct { + unsigned long sae_bear; + }; }; unsigned long gprs[10]; unsigned long back_chain;
diff --git a/arch/s390/include/uapi/Kbuild b/arch/s390/include/uapi/Kbuild new file mode 100644 index 0000000..81dcaaf --- /dev/null +++ b/arch/s390/include/uapi/Kbuild
@@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 + +# The ARM64 uapi headers under generated/uapi/arm64/ are private copies used +# internally by the s390 KVM ARM64 implementation. They contain #line +# directives and rewritten include paths and must not be shipped to userspace. +no-export-headers += arm64/%
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index f6dd2b6..f9f9c22 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c
@@ -70,6 +70,7 @@ int main(void) OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys); OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce); OFFSET(__SF_SIE_IRQ, stack_frame, sie_irq); + OFFSET(__SF_SAE_BEAR, stack_frame, sae_bear); DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame)); BLANK(); OFFSET(__SFUSER_BACKCHAIN, stack_frame_user, back_chain);
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 4242cd7..999dc7d 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S
@@ -245,6 +245,28 @@ EXPORT_SYMBOL(sie_exit) #endif +#if IS_ENABLED(CONFIG_KVM) +/* + * SAE calling convention: + * %r2 physical address of the SAE control block + */ +SYM_FUNC_START(sae) + stmg %r6,%r14,__SF_GPRS(%r15) # store kernel registers + STBEAR __SF_SAE_BEAR(%r15) # save breaking event address register + .insn rre,0xb9a50000,%r2,0 # Start Arm Execution + LBEAR __SF_SAE_BEAR(%r15) # restore breaking event address register + lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers + xgr %r0,%r0 # clear guest registers to + xgr %r1,%r1 # prevent speculative use + xgr %r2,%r2 + xgr %r3,%r3 + xgr %r4,%r4 + xgr %r5,%r5 + BR_EX %r14 +SYM_FUNC_END(sae) +EXPORT_SYMBOL(sae) +#endif + /* * SVC interrupt handler routine. System calls are synchronous events and * are entered with interrupts disabled.
diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index e33a3ec..6da55a15 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c
@@ -150,6 +150,7 @@ static void show_cpu_summary(struct seq_file *m, void *v) [HWCAP_NR_NNPA] = "nnpa", [HWCAP_NR_PCI_MIO] = "pcimio", [HWCAP_NR_SIE] = "sie", + [HWCAP_NR_SAE] = "sae", }; int i, cpu; @@ -254,6 +255,8 @@ static int __init setup_hwcaps(void) /* virtualization support */ if (sclp.has_sief2) elf_hwcap |= HWCAP_SIE; + if (sclp.has_aef) + elf_hwcap |= HWCAP_SAE; return 0; }
diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 8ea9dd7..abb9472 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c
@@ -16,6 +16,7 @@ #include <linux/swap.h> #include <linux/pagewalk.h> #include <linux/backing-dev.h> +#include <linux/slab.h> #include <linux/vmalloc.h> #include <asm/facility.h> #include <asm/sections.h> @@ -242,34 +243,38 @@ void uv_free_stor_var(void *stor_var) } EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm"); -static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data) -{ - struct page *page; - pte_t pte; - - page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!page) - return -ENOMEM; - pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL)); - set_pte(ptep, pte); - return 0; -} - void *uv_alloc_stor_var(unsigned long size) { + unsigned long i, nr_pages, addr; struct vm_struct *area; - unsigned long addr; + struct page **pages; size = PAGE_ALIGN(size); + nr_pages = size >> PAGE_SHIFT; area = get_vm_area(size, VM_SPARSE); if (!area) return NULL; + pages = kvcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL_ACCOUNT); + if (!pages) + goto free_area; + for (i = 0; i < nr_pages; i++) { + pages[i] = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!pages[i]) + goto free_pages; + } addr = (unsigned long)area->addr; - if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL)) - goto out; + if (vm_area_map_pages(area, addr, addr + size, pages)) + goto free_pages; + kvfree(pages); return area->addr; -out: - uv_free_stor_var(area->addr); +free_pages: + for (i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kvfree(pages); +free_area: + free_vm_area(area); return NULL; } EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm");
diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 8d3ee17..e7b525d 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig
@@ -25,22 +25,27 @@ select KVM_COMMON select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_INVALID_WAKEUPS - select HAVE_KVM_NO_POLL select KVM_VFIO select VIRT_XFER_TO_GUEST_WORK select KVM_MMU_LOCKLESS_AGING select KVM_GENERIC_PRE_FAULT_MEMORY help - Support hosting paravirtualized guest machines using the SIE - virtualization capability on the mainframe. This should work - on any 64bit machine. + Enable support for hosting virtualized, hardware-accelerated + virtual machines via two modules: kvm and kvm_arm64. - This module provides access to the hardware capabilities through - a character device node named /dev/kvm. + The kvm module provides access to host hardware-accelerated s390 + guests using the Start Interpretive Execution (SIE) instruction via a + character device node named /dev/kvm. This requires a 64-bit machine + (hardware model). - To compile this as a module, choose M here: the module - will be called kvm. + The kvm_arm64 module provides access to host hardware-accelerated + arm64 guests using the Start ARM Execution (SAE) instruction via a + character device node named /dev/kvm_arm64. This requires hardware + models that support the Arm execution facility (AEF). + + To compile this driver as a module, choose M here. The module will be + called kvm for the SIE acceleration and kvm_arm64 for the SAE + acceleration. If unsure, say N.
diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index a4e3875..380db44 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile
@@ -4,3 +4,4 @@ # Copyright IBM Corp. 2008 obj-$(CONFIG_KVM) += s390/ +obj-$(CONFIG_KVM) += arm64/
diff --git a/arch/s390/kvm/arm64/.gitignore b/arch/s390/kvm/arm64/.gitignore new file mode 100644 index 0000000..bd9d906 --- /dev/null +++ b/arch/s390/kvm/arm64/.gitignore
@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +generated/*
diff --git a/arch/s390/kvm/arm64/Makefile b/arch/s390/kvm/arm64/Makefile new file mode 100644 index 0000000..3f0fe5c --- /dev/null +++ b/arch/s390/kvm/arm64/Makefile
@@ -0,0 +1,110 @@ +# SPDX-License-Identifier: GPL-2.0 + +KVM := ../../../../virt/kvm +KVM_DEV_NAME = kvm_arm64 +KVM_DEV_MINOR = MISC_DYNAMIC_MINOR +include $(srctree)/virt/kvm/Makefile.kvm +include $(srctree)/arch/$(SRCARCH)/kvm/gmap/Makefile +include $(src)/Makefile.gen + +ccflags-y += -I $(src) -I $(objtree)/$(obj) -I$(srctree)/arch/$(SRCARCH)/kvm/gmap -DKVM_S390_ARM64 + +kvm-arm64-local := \ + arm.o \ + guest.o \ + handle_exit.o \ + inject_fault.o \ + mmio.o \ + mmu.o \ + reset.o + +kvm-arm64-obj := $(kvm-arm64-local) +kvm-arm64-obj += $(patsubst %.o,%-arm64.o,$(kvm-y)) +kvm-arm64-obj += $(patsubst %.o,%-arm64.o,$(gmap-y)) + +targets += $(kvm-arm64-obj) + +$(obj)/%-arm64.o: $(src)/%.c FORCE + @mkdir -p $(dir $@) + $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) + +# %-arm64.o stems don't match any kvm-arm64-y entry so Kbuild derives wrong +# KBUILD_MODNAME/KBUILD_MODFILE. Force include a generated header to override. +KVM_ARM64_MODNAME_H := $(obj)/generated/kvm_arm64_modname.h +ccflags-y += -include $(KVM_ARM64_MODNAME_H) + +$(addprefix $(obj)/,$(kvm-arm64-obj)): $(KVM_ARM64_MODNAME_H) +$(foreach sfx, i s ll lst, \ + $(eval $(patsubst %.o,%.$(sfx),$(addprefix $(obj)/,$(kvm-arm64-local))): $(KVM_ARM64_MODNAME_H))) + +targets += generated/kvm_arm64_modname.h + +define filechk_kvm_arm64_modname + echo '#ifndef __KVM_ARM64_MODNAME_H'; \ + echo '#define __KVM_ARM64_MODNAME_H'; \ + echo '#undef KBUILD_MODNAME'; \ + echo '#define KBUILD_MODNAME "$(KVM_DEV_NAME)"'; \ + echo '#undef __KBUILD_MODNAME'; \ + echo '#define __KBUILD_MODNAME $(KVM_DEV_NAME)'; \ + echo '#undef KBUILD_MODFILE'; \ + echo '#define KBUILD_MODFILE "$(obj)/kvm-arm64"'; \ + echo '#endif /* __KVM_ARM64_MODNAME_H */' +endef + +$(KVM_ARM64_MODNAME_H): FORCE + $(call filechk,kvm_arm64_modname) + +# Members are not in real-obj-y. Set part-of-builtin to enable instrumentation. +$(addprefix $(obj)/,$(kvm-arm64-obj)): private part-of-builtin := y + +ifeq ($(CONFIG_KVM),m) + +kvm-arm64-y = $(kvm-arm64-obj) + +else ifeq ($(CONFIG_KVM),y) + +prereq-o-cmd = $(foreach o, $(filter %.o, $^), $(dir $(o)).$(notdir $(o)).cmd) +cmd_gen_symversions_o = \ + grep --no-filename "^\#SYMVER" $(prereq-o-cmd) >> $(dot-target).cmd || true + +cmd_ld_o_o = $(cmd_ld) +define rule_ld_o_o + $(call cmd_and_savecmd,ld) + $(call cmd,gen_symversions_o) +endef + +LDFLAGS_kvm-unnamespaced.o := -r +$(obj)/kvm-unnamespaced.o: $(addprefix $(obj)/,$(kvm-arm64-obj)) FORCE + $(call if_changed_rule,ld_o_o) + +quiet_cmd_nm_filter = NMFLTR $@ + cmd_nm_filter = $(NM) --defined-only --format=just-symbols $< | sort -u > $@ + +$(obj)/kvm_symbol_list: $(obj)/kvm-unnamespaced.o FORCE + $(call if_changed,nm_filter) + +# cmd_oc_o_o carries -DKBUILD_MODFILE so generate_builtin_ranges.awk +# can attribute kvm-namespaced.o to kvm_arm64 via its .cmd file. +cmd_oc_o_o = $(cmd_objcopy) -DKBUILD_MODFILE='"$(obj)/kvm-arm64"' +define rule_oc_o_o + $(call cmd,objcopy) + @printf '%s\n' 'savedcmd_$@ := $(call make-cmd,oc_o_o)' > $(dot-target).cmd + $(call cmd,gen_objtooldep) + $(call cmd,gen_symversions_o) +endef + +OBJCOPYFLAGS_kvm-namespaced.o := -O $(LD_BFD) --localize-symbols=$(obj)/kvm_symbol_list +$(obj)/kvm-namespaced.o: $(obj)/kvm-unnamespaced.o $(obj)/kvm_symbol_list FORCE + $(call if_changed_rule,oc_o_o) + +kvm-arm64-y = kvm-namespaced.o + +targets += kvm-unnamespaced.o kvm_symbol_list kvm-namespaced.o + +endif + +clean-files += kvm-unnamespaced.o kvm_symbol_list kvm-namespaced.o + +obj-$(CONFIG_KVM) += kvm-arm64.o +
diff --git a/arch/s390/kvm/arm64/Makefile.gen b/arch/s390/kvm/arm64/Makefile.gen new file mode 100644 index 0000000..9c33b06 --- /dev/null +++ b/arch/s390/kvm/arm64/Makefile.gen
@@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Extracts ARM64 marked sections to .inc files that are included by main source +# files + +# List of ARM64 C files to extract +ARM64_CFILES := \ + arm.c \ + guest.c \ + mmio.c + +quiet_cmd_extract_inc = GEN $@ + cmd_extract_inc = mkdir -p $(dir $@); \ + $(AWK) -f $(src)/copy-arm64c.awk $< > $@ || \ + { echo "Error: No or unbalanced ARM64_S390_COMMON markers in $<" >&2; rm -f $@; exit 1; } + +ARM64_INC_FILES := $(foreach cfile,$(ARM64_CFILES),generated/$(basename $(cfile)).inc) + +$(addprefix $(obj)/,$(ARM64_INC_FILES)): $(obj)/generated/%.inc: $(srctree)/arch/arm64/kvm/%.c $(src)/copy-arm64c.awk FORCE + $(call if_changed,extract_inc) + +targets += $(ARM64_INC_FILES) +clean-files += generated/*.inc + +$(foreach cfile,$(basename $(ARM64_CFILES)),\ + $(eval $(obj)/$(cfile).o $(obj)/$(cfile).i $(obj)/$(cfile).s \ + $(obj)/$(cfile).ll $(obj)/$(cfile).lst: \ + $(obj)/generated/$(cfile).inc))
diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c new file mode 100644 index 0000000..bd9d852 --- /dev/null +++ b/arch/s390/kvm/arm64/arm.c
@@ -0,0 +1,754 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/miscdevice.h> +#include <linux/kvm.h> +#include <linux/kvm_types.h> +#include <linux/kvm_host.h> +#include <linux/cleanup.h> +#include <linux/fpu.h> + +#include <asm/access-regs.h> + +#include <arm64/kvm_emulate.h> +#include <arm64/sysreg.h> + +#include <gmap.h> +#include <kvm_mmu.h> + +#include "arm.h" +#include "handle_exit.h" + +#define CREATE_TRACE_POINTS +#include "trace.h" + +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu); + +#define __INCL_GEN_ARM_FILE +#include "generated/arm.inc" +#undef __INCL_GEN_ARM_FILE + +#define KVM_PHYS_SHIFT (40) + +int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +{ + int ret; + + switch (ext) { + case KVM_CAP_NR_VCPUS: + case KVM_CAP_MAX_VCPUS: + case KVM_CAP_MAX_VCPU_ID: + ret = KVM_MAX_VCPUS; + break; + case KVM_CAP_ARM_VM_IPA_SIZE: + ret = get_kvm_ipa_limit(); + break; + case KVM_CAP_IOEVENTFD: + case KVM_CAP_PRE_FAULT_MEMORY: + ret = 1; + break; + default: + ret = 0; + } + + return ret; +} + +static u64 kvm_max_guest_address(void) +{ + u64 max_addr; + + max_addr = min_t(u64, TASK_SIZE_MAX, sclp.hamax); + max_addr = max_t(u64, max_addr, SZ_1G - 1); + return ALIGN_DOWN(max_addr + 1, SZ_1G) - 1; +} + +static int kvm_gmap_init(struct kvm *kvm) +{ + struct crst_table *table; + + kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.guest_phys_size)); + + if (!kvm->arch.gmap) + return -ENOMEM; + + /* arm64 (on s390) do not have pfault */ + clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags); + set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + + table = dereference_asce(kvm->arch.gmap->asce); + crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val); + + return 0; +} + +static int kvm_vm_type_ipa_size_shift(unsigned long type) +{ + int phys_shift; + + phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type); + if (phys_shift) { + if (phys_shift > get_kvm_ipa_limit() || + phys_shift < ARM64_MIN_PARANGE_BITS) + return -EINVAL; + } else { + phys_shift = KVM_PHYS_SHIFT; + if (phys_shift > get_kvm_ipa_limit()) { + pr_warn_once("%s using unsupported default IPA limit\n", + current->comm); + return -EINVAL; + } + } + + return phys_shift; +} + +int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) +{ + char debug_name[32]; + int ret; + + if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) + return -EINVAL; + + ret = kvm_vm_type_ipa_size_shift(type); + if (ret < 0) + return ret; + kvm->arch.guest_phys_size = 1UL << ret; + + mutex_init(&kvm->arch.config_lock); + bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); + + snprintf(debug_name, sizeof(debug_name), "kvm-arm64-%u-%u", + current->pid, get_random_u32()); + kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long)); + if (!kvm->arch.dbf) + return -ENOMEM; + debug_register_view(kvm->arch.dbf, &debug_sprintf_view); + + ret = kvm_gmap_init(kvm); + if (ret) + goto out_err; + kvm->arch.mem_limit = kvm->arch.guest_phys_size; + + VM_EVENT(kvm, 3, "vm created with type %lu", type); + return 0; + +out_err: + debug_unregister(kvm->arch.dbf); + + return ret; +} + +vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + return -EINVAL; +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kvm_destroy_vcpus(kvm); + debug_unregister(kvm->arch.dbf); + kvm->arch.gmap = gmap_put(kvm->arch.gmap); +} + +u32 get_kvm_ipa_limit(void) +{ + return fls64(kvm_max_guest_address() + 1) - 1; +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ + struct kvm_sae_block *sae_block = &vcpu->arch.sae_block; + struct kvm_sae_save_area *save_area = &vcpu->arch.save_area; + + spin_lock_init(&vcpu->arch.mp_state_lock); + + /* Force users to call KVM_ARM_VCPU_INIT */ + vcpu_clear_flag(vcpu, VCPU_INITIALIZED); + + vcpu->arch.mc = kvm_s390_new_mmu_cache(); + if (!vcpu->arch.mc) + return -ENOMEM; + + save_area->sdo = virt_to_phys(sae_block); + + sae_block->hbasce = vcpu->kvm->arch.gmap->asce.val; + sae_block->mso = 0L; + sae_block->msl = vcpu->kvm->arch.mem_limit - 1; + sae_block->save_area = virt_to_phys(save_area); + + VM_EVENT(vcpu->kvm, 3, + "create cpu %d at 0x%p, sae block at 0x%p, satellite at 0x%p", + vcpu->vcpu_id, vcpu, &vcpu->arch.sae_block, + &vcpu->arch.save_area); + return 0; +} + +void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) +{ +} + +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + kvm_s390_free_mmu_cache(vcpu->arch.mc); + + VCPU_EVENT(vcpu, 3, "%s", "free cpu"); +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + save_access_regs(&vcpu->arch.host_acrs[0]); + vcpu->cpu = cpu; + + lasrm(&vcpu->arch.save_area); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + stiasrm(&vcpu->arch.save_area); + + vcpu->cpu = -1; + restore_access_regs(&vcpu->arch.host_acrs[0]); +} + +int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + *mp_state = READ_ONCE(vcpu->arch.mp_state); + return 0; +} + +int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return 0; +} + +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) +{ + unsigned long features = KVM_S390_ARM64_IMPL_FEATURES; + + return features; +} + +bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) +{ + return vcpu_mode_priv(vcpu); +} + +int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + if (!kvm_arm_vcpu_is_finalized(vcpu)) + return -EPERM; + + if (likely(READ_ONCE(vcpu->pid))) + return 0; + + scoped_guard(mutex, &kvm->arch.config_lock) + set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags); + + return 0; +} + +/** + * check_vcpu_requests - check and handle pending vCPU requests + * @vcpu: the VCPU pointer + * + * Return: 1 if we should enter the guest + * 0 if we should exit to userspace + * < 0 if we should exit to userspace, where the return value indicates + * an error + */ +static int check_vcpu_requests(struct kvm_vcpu *vcpu) +{ + if (kvm_request_pending(vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) + return -EIO; + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) + kvm_reset_vcpu(vcpu); + /* + * Clear IRQ_PENDING requests that were made to guarantee + * that a VCPU sees new virtual interrupts. + */ + kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); + + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) + asce_flush_tlb(vcpu->kvm->arch.gmap->asce); + } + + return 1; +} + +static int kvm_vcpu_initialize(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + unsigned long features = init->features[0]; + struct kvm *kvm = vcpu->kvm; + + guard(mutex)(&kvm->arch.config_lock); + if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) && + kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES); + + kvm_reset_vcpu(vcpu); + + set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags); + vcpu_set_flag(vcpu, VCPU_INITIALIZED); + + return 0; +} + +static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + int ret; + + if (init->target != KVM_ARM_TARGET_GENERIC_V8) + return -EINVAL; + + ret = kvm_vcpu_init_check_features(vcpu, init); + if (ret) + return ret; + + if (!kvm_vcpu_initialized(vcpu)) + return kvm_vcpu_initialize(vcpu, init); + + if (kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + kvm_reset_vcpu(vcpu); + + return 0; +} + +static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, + struct kvm_vcpu_init *init) +{ + struct kvm_sae_save_area *save_area = &vcpu->arch.save_area; + struct kvm_sae_block *sae_block = &vcpu->arch.sae_block; + int ret; + + sae_block->save_area = virt_to_phys(save_area); + save_area->sdo = virt_to_phys(sae_block); + + ret = kvm_vcpu_set_target(vcpu, init); + if (ret) + return ret; + + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); + } + + return 0; +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, + bool line_status) +{ + /* stub for now*/ + return -EINVAL; +} + +static void adjust_pc(struct kvm_vcpu *vcpu) +{ + if (vcpu_get_flag(vcpu, INCREMENT_PC)) { + kvm_skip_instr(vcpu); + vcpu_clear_flag(vcpu, INCREMENT_PC); + } +} + +static noinstr void arm_vcpu_enter_exit(phys_addr_t sae_block) +{ + guest_timing_enter_irqoff(); + guest_state_enter_irqoff(); + + /* + * External interrupts may occur while the SAE instruction is executing + * and must preempt guest execution to maintain responsiveness of the + * host. + * + * As a temporary workaround, run SAE with interrupts enabled so that any + * interrupt is taken via the normal host interrupt path rather than + * being subsumed inside the SAE window. + */ + raw_local_irq_enable(); + sae(sae_block); + raw_local_irq_disable(); + + guest_state_exit_irqoff(); + guest_timing_exit_irqoff(); +} + +/** kvm_arch_vcpu_ioctl_run() - run arm64 vCPU + * + * Execute arm64 guest instructions using SAE. + * + * Returns: + * 1 enter the guest (should not be observed by userspace) + * 0 exit to userspace + * < 0 exit to userspace, where the return value indicates an error + * + * + */ +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) +{ + DECLARE_KERNEL_FPU_ONSTACK32(fpu_save); + struct kvm_run *kvm_run = vcpu->run; + int ret; + + if (kvm_run->exit_reason == KVM_EXIT_MMIO) { + ret = kvm_handle_mmio_return(vcpu); + if (ret <= 0) + return ret; + } + + vcpu_load(vcpu); + + kernel_fpu_begin(&fpu_save, KERNEL_FPC | KERNEL_VXR); + load_vx_regs((vcpu->arch.ctxt.vregs)); + + if (!vcpu->wants_to_run) { + ret = -EINTR; + goto out; + } + + kvm_sigset_activate(vcpu); + + might_fault(); + + ret = 1; + kvm_run->exit_reason = KVM_EXIT_UNKNOWN; + while (ret > 0) { + ret = kvm_xfer_to_guest_mode_handle_work(vcpu); + if (!ret) + ret = 1; + + if (ret > 0) + ret = check_vcpu_requests(vcpu); + + vcpu->arch.sae_block.icptr = 0; + + local_irq_disable(); + xfer_to_guest_mode_prepare(); + /* + * The full memory barrier in smp_store_mb() orders the mode + * store against the kvm_request_pending() check below, so + * any pending kick is visible before we decide to enter the + * guest. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + if (ret <= 0 || kvm_request_pending(vcpu) || + xfer_to_guest_mode_work_pending()) { + vcpu->mode = OUTSIDE_GUEST_MODE; + local_irq_enable(); + continue; + } + + trace_kvm_entry(*vcpu_pc(vcpu)); + + adjust_pc(vcpu); + + arm_vcpu_enter_exit(virt_to_phys(&vcpu->arch.sae_block)); + + vcpu->mode = OUTSIDE_GUEST_MODE; + vcpu->stat.exits++; + + local_irq_enable(); + trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); + + ret = handle_exit(vcpu); + } + + kvm_sigset_deactivate(vcpu); +out: + if (unlikely(vcpu_get_flag(vcpu, INCREMENT_PC))) + adjust_pc(vcpu); + + save_vx_regs(vcpu->arch.ctxt.vregs); + kernel_fpu_end(&fpu_save, KERNEL_FPC | KERNEL_VXR); + vcpu_put(vcpu); + + return ret; +} + +static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + struct kvm_device_attr attr; + int ret; + + switch (ioctl) { + case KVM_ARM_VCPU_INIT: { + struct kvm_vcpu_init init; + + ret = -EFAULT; + if (copy_from_user(&init, argp, sizeof(init))) + break; + + ret = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); + break; + } + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: { + struct kvm_one_reg reg; + + ret = -ENOEXEC; + if (unlikely(!kvm_vcpu_initialized(vcpu))) + break; + + ret = -EFAULT; + if (copy_from_user(®, argp, sizeof(reg))) + break; + + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) + kvm_reset_vcpu(vcpu); + + if (ioctl == KVM_SET_ONE_REG) + ret = kvm_arm_set_reg(vcpu, ®); + else + ret = kvm_arm_get_reg(vcpu, ®); + break; + } + case KVM_GET_REG_LIST: { + struct kvm_reg_list __user *user_list = argp; + struct kvm_reg_list reg_list; + unsigned int n; + + ret = -ENOEXEC; + if (unlikely(!kvm_vcpu_initialized(vcpu))) + break; + ret = -EPERM; + if (!kvm_arm_vcpu_is_finalized(vcpu)) + break; + ret = -EFAULT; + if (copy_from_user(®_list, user_list, sizeof(reg_list))) + break; + n = reg_list.n; + reg_list.n = kvm_arm_num_regs(vcpu); + if (copy_to_user(user_list, ®_list, sizeof(reg_list))) + break; + ret = -E2BIG; + if (n < reg_list.n) + break; + ret = kvm_arm_copy_reg_indices(vcpu, user_list->reg); + break; + } + case KVM_ARM_VCPU_FINALIZE: { + int what; + + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + if (get_user(what, (const int __user *)argp)) + return -EFAULT; + + ret = kvm_arm_vcpu_finalize(vcpu, what); + break; + } + case KVM_SET_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_set_attr(vcpu, &attr); + break; + } + case KVM_GET_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_get_attr(vcpu, &attr); + break; + } + case KVM_HAS_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_has_attr(vcpu, &attr); + break; + } + default: + ret = -EINVAL; + } + + return ret; +} + +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + return s390_kvm_mmu_get_dirty_log(kvm, log); +} + +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return gmap_age_gfn(kvm->arch.gmap, range->start, range->end); +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + gfn_t last_gfn = memslot->base_gfn + memslot->npages; + + scoped_guard(read_lock, &kvm->mmu_lock) + gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn); +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ +} + +bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end); +} + +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + + switch (ioctl) { + case KVM_ARM_PREFERRED_TARGET: { + struct kvm_vcpu_init init = { + .target = KVM_ARM_TARGET_GENERIC_V8, + }; + + if (copy_to_user(argp, &init, sizeof(init))) + return -EFAULT; + + return 0; + } + + default: + return -EINVAL; + } +} + +bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) +{ + return false; +} + +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status) +{ + return -EINVAL; +} + +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + return -EINVAL; +} + +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return 0; +} + +long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + return -ENOIOCTLCMD; +} + +static int __init kvm_s390_arm64_init(void) +{ + if (!sclp.has_aef) + return -ENODEV; + + return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); +} + +static __exit void kvm_s390_arm64_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_s390_arm64_init); +module_exit(kvm_s390_arm64_exit);
diff --git a/arch/s390/kvm/arm64/arm.h b/arch/s390/kvm/arm64/arm.h new file mode 100644 index 0000000..7602306 --- /dev/null +++ b/arch/s390/kvm/arm64/arm.h
@@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_KVM_ARM64_H +#define ARCH_S390_KVM_ARM64_H + +#include <linux/compiler_attributes.h> +#include <linux/kvm_host.h> + +#define VM_EVENT(d_kvm, d_loglevel, d_string, d_args...) \ + debug_sprintf_event((d_kvm)->arch.dbf, d_loglevel, \ + __stringify(KVM_DEV_NAME) ": " d_string "\n", \ + d_args) + +#define VCPU_EVENT(d_vcpu, d_loglevel, d_string, d_args...) \ + debug_sprintf_event((d_vcpu)->kvm->arch.dbf, d_loglevel, \ + __stringify(KVM_DEV_NAME) " %02d[%016llx-%016llx]: " d_string "\n", \ + (d_vcpu)->vcpu_id, (d_vcpu)->arch.sae_block.pstate, \ + (d_vcpu)->arch.sae_block.pc, d_args) + +static __always_inline int kvm_is_ucontrol(struct kvm *kvm) +{ + return 0; +} + +static __always_inline int __kvm_s390_pv_destroy_page(struct page *page) +{ + return 0; +} + +static __always_inline void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end) +{ +} + +static __always_inline int kvm_s390_pv_get_handle(struct kvm *kvm) +{ + return 0; +} + +static __always_inline int kvm_s390_is_migration_mode(struct kvm *kvm) +{ + return false; +} + +static __always_inline bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) +{ + return false; +} + +static __always_inline void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old) +{ +} + +/* should never be called */ +static __always_inline int kvm_s390_vm_stop_migration(struct kvm *kvm) +{ + return -EINVAL; +} + +#endif /* ARCH_S390_KVM_ARM64_H */
diff --git a/arch/s390/kvm/arm64/copy-arm64c.awk b/arch/s390/kvm/arm64/copy-arm64c.awk new file mode 100644 index 0000000..640a7bc --- /dev/null +++ b/arch/s390/kvm/arm64/copy-arm64c.awk
@@ -0,0 +1,82 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Extract marked sections from ARM64 C files for sharing with s390 KVM +# +# Usage: copy-arm64c.awk <input_file> +# +# Extracts all sections between start/end markers. If no markers found, signals failure. + +BEGIN { + # Constants + start_pattern = "^#if ARM64_S390_COMMON == 1$" + end_pattern = "^#endif /\\* ARM64_S390_COMMON == 1 \\*/$" + + # State variables + copying = found_marker = 0 + file_header_done = 0 + depth = 0 +} + +!file_header_done { + if (/^\/\*/ || /^\/\/ SPDX-License-Identifier:/) { + print + next + } + if (/[[:space:]]\*([[:space:]]|$)/) { + print + next + } + if (/\*\//) { + print " *" + } else { + print "/*" + } + + print " * This file was automatically generated from " FILENAME + print " * Do not modify this file directly." + print " */" + print "" + print "#ifndef __INCL_GEN_ARM_FILE" + print "#error do not include this file directly." + print "#endif /* __INCL_GEN_ARM_FILE */" + print "#undef __INCL_GEN_ARM_FILE" + print "" + + file_header_done = 1 +} + +$0 ~ start_pattern { + if (copying) { error = 1; exit } + depth = copying = found_marker = 1 + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " (NR + 1) " \"" FILENAME "\"" + next +} + +copying && /^[ \t]*#[ \t]*if(def|ndef)?([ \t]|$)/ { depth++ } + +$0 ~ /^[ \t]*#[ \t]*endif/ { + depth-- + if (depth == 0) { + if ($0 ~ end_pattern) { + copying = 0 + next + } + error = 1; exit + } +} + +/ARM64_S390_COMMON/ { error = 1; exit } + +copying { + gsub(/#include <asm\//, "#include <arm64/") + gsub(/#include <uapi\/asm\//, "#include <uapi/arm64/") + print + next +} + +END { + exit error || !found_marker || copying +}
diff --git a/arch/s390/kvm/arm64/guest.c b/arch/s390/kvm/arm64/guest.c new file mode 100644 index 0000000..4ea6fe2 --- /dev/null +++ b/arch/s390/kvm/arm64/guest.c
@@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/kvm_host.h> +#include <linux/kvm.h> + +#include <arm64/kvm_emulate.h> +#include <arm64/kvm_nested.h> + +#define __INCL_GEN_ARM_FILE +#include "generated/guest.inc" +#undef __INCL_GEN_ARM_FILE + +const struct kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS() +}; + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + /* ARM64 stats */ + STATS_DESC_COUNTER(VCPU, hvc_exit_stat), + STATS_DESC_COUNTER(VCPU, wfe_exit_stat), + STATS_DESC_COUNTER(VCPU, wfi_exit_stat), + STATS_DESC_COUNTER(VCPU, mmio_exit_user), + STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, exits), + /* GMAP stats */ + STATS_DESC_COUNTER(VCPU, pfault_sync), +}; + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), +}; + +int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + int ret; + + ret = copy_core_reg_indices(vcpu, uindices); + if (ret < 0) + return ret; + uindices += ret; + + return 0; +} + +unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) +{ + return num_core_regs(vcpu); +} + +int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + /* We currently use nothing arch-specific in upper 32 bits */ + if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) + return -EINVAL; + + switch (reg->id & KVM_REG_ARM_COPROC_MASK) { + case KVM_REG_ARM_CORE: + return get_core_reg(vcpu, reg); + default: + return -EINVAL; + } +} + +int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + /* We currently use nothing arch-specific in upper 32 bits */ + if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) + return -EINVAL; + + switch (reg->id & KVM_REG_ARM_COPROC_MASK) { + case KVM_REG_ARM_CORE: + return set_core_reg(vcpu, reg); + default: + return -EINVAL; + } +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + return -EINVAL; +}
diff --git a/arch/s390/kvm/arm64/handle_exit.c b/arch/s390/kvm/arm64/handle_exit.c new file mode 100644 index 0000000..e0c7773 --- /dev/null +++ b/arch/s390/kvm/arm64/handle_exit.c
@@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kvm_host.h> + +#include <arm64/esr.h> +#include <arm64/kvm_emulate.h> + +#include "handle_exit.h" + +typedef int (*exit_handle_fn)(struct kvm_vcpu *); + +#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 +#define PSCI_RET_NOT_SUPPORTED -1 +#define PSCI_RET_INTERNAL_FAILURE -6 +/* + * Temporary smc/hvc handler. Non-compliant implementation (features missing). + * Implements only system off so that test programs are able to end their execution. + * Multi-vCPU power state management and KVM_REQ_SLEEP handling are deferred + * to future SMP series. + */ +static int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) +{ + u32 func_id = vcpu_get_reg(vcpu, 0); + u64 val = PSCI_RET_NOT_SUPPORTED; + int ret = 1; + + if (func_id == PSCI_0_2_FN_SYSTEM_OFF) { + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + } + memset(&vcpu->run->system_event, 0, + sizeof(vcpu->run->system_event)); + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SHUTDOWN; + vcpu->run->system_event.ndata = 1; + vcpu->run->system_event.data[0] = 0; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + val = PSCI_RET_INTERNAL_FAILURE; + ret = 0; + } + vcpu_set_reg(vcpu, 0, val); + + return ret; +} + +static int handle_hvc(struct kvm_vcpu *vcpu) +{ + vcpu->stat.hvc_exit_stat++; + return kvm_smccc_call_handler(vcpu); +} + +static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + + kvm_pr_unimpl("Unknown exception class: esr: %#016llx -- %s\n", + esr, esr_get_class_string(esr)); + + kvm_inject_undefined(vcpu); + return 1; +} + +static exit_handle_fn arm_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec, + [ESR_ELx_EC_HVC64] = handle_hvc, + [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, + [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, +}; + +static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + u8 esr_ec = ESR_ELx_EC(esr); + + return arm_exit_handlers[esr_ec]; +} + +/* + * We may be single-stepping an emulated instruction. If the emulation + * has been completed in the kernel, we can return to userspace with a + * KVM_EXIT_DEBUG, otherwise userspace needs to complete its + * emulation first. + */ +static int handle_trap_exceptions(struct kvm_vcpu *vcpu) +{ + exit_handle_fn exit_handler; + + exit_handler = kvm_get_exit_handler(vcpu); + return exit_handler(vcpu); +} + +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int handle_exit(struct kvm_vcpu *vcpu) +{ + u8 icptr = vcpu->arch.sae_block.icptr; + int ret = 1; + + switch (icptr) { + case SAE_ICPTR_SPURIOUS: + break; + case SAE_ICPTR_VALIDITY: + WARN_ONCE(true, "SAE: validity intercept. vir: 0x%04x", + vcpu->arch.sae_block.vir); + vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; + ret = 0; + break; + case SAE_ICPTR_GUEST_ADDRESS_SIZE: + case SAE_ICPTR_HOST_ACCESS_EXCEPTION: + case SAE_ICPTR_SYNCHRONOUS_EXCEPTION: + ret = handle_trap_exceptions(vcpu); + break; + default: + vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; + ret = 0; + } + return ret; +} + +/* manually copied from arch/arm64/kernel/traps.c */ +static const char * const esr_class_str[] = { + [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", + [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", + [ESR_ELx_EC_WFx] = "WFI/WFE", + [ESR_ELx_EC_FP_ASIMD] = "ASIMD", + [ESR_ELx_EC_PAC] = "PAC", + [ESR_ELx_EC_BTI] = "BTI", + [ESR_ELx_EC_ILL] = "PSTATE.IL", + [ESR_ELx_EC_SVC64] = "SVC (AArch64)", + [ESR_ELx_EC_HVC64] = "HVC (AArch64)", + [ESR_ELx_EC_SMC64] = "SMC (AArch64)", + [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", + [ESR_ELx_EC_SVE] = "SVE", + [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB", + [ESR_ELx_EC_FPAC] = "FPAC", + [ESR_ELx_EC_SME] = "SME", + [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", + [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", + [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", + [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", + [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", + [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", + [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", + [ESR_ELx_EC_MOPS] = "MOPS", + [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", + [ESR_ELx_EC_GCS] = "Guarded Control Stack", + [ESR_ELx_EC_SERROR] = "SError", + [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", + [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", + [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", + [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", + [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", + [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", + [ESR_ELx_EC_BRK64] = "BRK (AArch64)", +}; + +const char *esr_get_class_string(unsigned long esr) +{ + return esr_class_str[ESR_ELx_EC(esr)]; +}
diff --git a/arch/s390/kvm/arm64/handle_exit.h b/arch/s390/kvm/arm64/handle_exit.h new file mode 100644 index 0000000..65ae587 --- /dev/null +++ b/arch/s390/kvm/arm64/handle_exit.h
@@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef KVM_ARM64_HANDLE_EXIT_H +#define KVM_ARM64_HANDLE_EXIT_H + +#include <linux/kvm_host.h> + +int handle_exit(struct kvm_vcpu *vcpu); + +#endif /* KVM_ARM64_HANDLE_EXIT_H */
diff --git a/arch/s390/kvm/arm64/inject_fault.c b/arch/s390/kvm/arm64/inject_fault.c new file mode 100644 index 0000000..425dbea --- /dev/null +++ b/arch/s390/kvm/arm64/inject_fault.c
@@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <arm64/kvm_emulate.h> + +/** + * kvm_inject_undefined - inject an undefined instruction into the guest + * @vcpu: The vCPU in which to inject the exception + * + * It is assumed that this code is called from the VCPU thread and that the + * VCPU therefore is not currently executing guest code. + */ +void kvm_inject_undefined(struct kvm_vcpu *vcpu) +{ + /* Stub until s390 supports arm64 sysregs TODO sysregs*/ +} + +int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) +{ + /* Stub until s390 supports arm64 sysregs TODO sysregs*/ + return 1; +}
diff --git a/arch/s390/kvm/arm64/mmio.c b/arch/s390/kvm/arm64/mmio.c new file mode 100644 index 0000000..908a6b4 --- /dev/null +++ b/arch/s390/kvm/arm64/mmio.c
@@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kvm_host.h> +#include <linux/bitfield.h> + +#include <trace/events/kvm.h> + +#include <arm64/kvm_emulate.h> +#include <arm64/sysreg.h> + +#include "trace.h" + +#define __INCL_GEN_ARM_FILE +#include "generated/mmio.inc" +#undef __INCL_GEN_ARM_FILE
diff --git a/arch/s390/kvm/arm64/mmu.c b/arch/s390/kvm/arm64/mmu.c new file mode 100644 index 0000000..31e0d03 --- /dev/null +++ b/arch/s390/kvm/arm64/mmu.c
@@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/kvm_host.h> + +#include <arm64/kvm_emulate.h> + +#include <faultin.h> + +#include "trace.h" + +static inline bool kvm_s390_cur_gmap_fault_is_write(struct kvm_vcpu *vcpu) +{ + return (vcpu->arch.sae_block.hai.pic & PGM_INT_CODE_MASK) == PGM_PROTECTION || + vcpu->arch.sae_block.hai.teid.fsi == TEID_FSI_STORE; +} + +/* + * user_mem_abort() - handle a dat fault for the gmap of a vcpu + * + * Return: 0 on success, < 0 in case of error. + * Context: The mm lock must not be held before calling. May sleep. + */ +static int user_mem_abort(struct kvm_vcpu *vcpu, gpa_t fault_ipa, + struct kvm_memory_slot *slot, hva_t hva) +{ + struct guest_fault f = { }; + int ret; + + if (kvm_s390_cur_gmap_fault_is_write(vcpu)) + f.write_attempt = FOLL_WRITE; + f.gfn = gpa_to_gfn(fault_ipa); + + ret = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (ret <= 0) + return ret; + if (ret == PGM_ADDRESSING) + /* + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + return -ENOEXEC; + KVM_BUG_ON(ret, vcpu->kvm); + return -EINVAL; +} + +static int kvm_handle_pic(struct kvm_vcpu *vcpu, bool *translation) +{ + switch (kvm_vcpu_fault_pic(vcpu)) { + /* expected cases: */ + case PGM_ASCE_TYPE: + case PGM_REGION_FIRST_TRANS: + case PGM_REGION_SECOND_TRANS: + case PGM_REGION_THIRD_TRANS: + case PGM_SEGMENT_TRANSLATION: + case PGM_PAGE_TRANSLATION: + *translation = true; + break; + case PGM_PROTECTION: + break; + /* unexpected cases: */ + case 0: + KVM_BUG(1, vcpu->kvm, "On MMU fault path but no fault occurred"); + return -EFAULT; + default: + KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx", + vcpu->arch.sae_block.hai.pic, vcpu->arch.sae_block.hai.teid.val); + send_sig(SIGSEGV, current, 0); + return -EFAULT; + } + + return 0; +} + +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) +{ + struct kvm_memory_slot *memslot; + bool translation = false; + phys_addr_t fault_ipa; + unsigned long hva; + unsigned long esr; + bool write_fault; + bool guest_size_err; + bool writable; + bool is_iabt; + int ret; + gfn_t gfn; + + esr = kvm_vcpu_get_esr(vcpu); + fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); + is_iabt = kvm_vcpu_trap_is_iabt(vcpu); + guest_size_err = vcpu->arch.sae_block.icptr == SAE_ICPTR_GUEST_ADDRESS_SIZE; + + if (guest_size_err) { + translation = true; + } else { + ret = kvm_handle_pic(vcpu, &translation); + if (ret) + return ret; + } + + if (translation) { + /* + * For both cases: + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + if (fault_ipa >= BIT_ULL(get_kvm_ipa_limit())) + return -ENOEXEC; + + if (fault_ipa >= vcpu->kvm->arch.guest_phys_size) + return -ENOEXEC; + } + + trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), + kvm_vcpu_get_hfar(vcpu), fault_ipa); + + kvm_vcpu_srcu_read_lock(vcpu); + + gfn = fault_ipa >> PAGE_SHIFT; + + memslot = gfn_to_memslot(vcpu->kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + write_fault = kvm_s390_cur_gmap_fault_is_write(vcpu); + if (kvm_is_error_hva(hva) || (write_fault && !writable)) { + ret = -ENOEXEC; + /* + * The guest has put either its instructions or its page-tables + * somewhere it shouldn't have. Userspace won't be able to do + * anything about this (there's no syndrome for a start). + * + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + if (is_iabt) + goto out_unlock; + + if (kvm_vcpu_abt_iss1tw(vcpu)) { + /* + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + goto out_unlock; + } + + /* + * Check for a cache maintenance operation. Assume the guest is + * cautious and skip instruction + */ + if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) { + kvm_incr_pc(vcpu); + ret = 1; + goto out_unlock; + } + + /* + * The IPA is reported as [MAX:12], so we need to + * complement it with the bottom 12 bits from the + * faulting VA. This is always 12 bits, irrespective + * of the page size. + */ + fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); + ret = io_mem_abort(vcpu, fault_ipa); + goto out_unlock; + } + + /* Userspace should not be able to register out-of-bounds IPAs */ + VM_BUG_ON(fault_ipa >= vcpu->kvm->arch.guest_phys_size); + /* + * Proper guest size faults have been injected. + * In theory it's fine to have device memory higher than MSL, + * even if not currently possible, but that would have been handled above. + * So if we get here with a guest size intercept, we have a bug somewhere. + */ + VM_BUG_ON(guest_size_err); + + ret = user_mem_abort(vcpu, fault_ipa, memslot, hva); + if (!ret) + ret = 1; +out_unlock: + kvm_vcpu_srcu_read_unlock(vcpu); + return ret; +} + +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) +{ + struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), }; + gpa_t end; + int rc; + + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc == PGM_ADDRESSING) + return -ENOENT; + if (rc > 0) + return -EIO; + if (rc < 0) + return rc; + + if (f.ptep) + return PAGE_SIZE; + + end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE); + return min(range->size, end - range->gpa); +} + +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + return 0; +}
diff --git a/arch/s390/kvm/arm64/reset.c b/arch/s390/kvm/arm64/reset.c new file mode 100644 index 0000000..e4cdce1 --- /dev/null +++ b/arch/s390/kvm/arm64/reset.c
@@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kvm_host.h> +#include <linux/fpu.h> + +#include <arm64/kvm_emulate.h> +#include <arm64/kvm_nested.h> + +bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) +{ + return true; +} + +static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hcr_el2 = HCR_EL2_E2H | HCR_EL2_RW | HCR_EL2_AMO | + HCR_EL2_IMO | HCR_EL2_FMO | HCR_EL2_PTW; + /* traps */ + vcpu->arch.hcr_el2 |= HCR_EL2_TSC | HCR_EL2_TID1 | HCR_EL2_TID2 | + HCR_EL2_TID3 | HCR_EL2_TID4 | HCR_EL2_TID5 | + HCR_EL2_TIDCP | HCR_EL2_TLOR; +} + +void kvm_reset_vcpu(struct kvm_vcpu *vcpu) +{ + struct vcpu_reset_state reset_state; + bool loaded; + + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + reset_state = vcpu->arch.reset_state; + vcpu->arch.reset_state.reset = false; + } + + /* + * Disable preemption around the vcpu reset as we might otherwise race with + * preempt notifiers which call stiasrm/lasrm from put/load + */ + preempt_disable(); + + /* The reset must run with an unloaded save area */ + loaded = vcpu_is_loaded(vcpu); + if (loaded) + vcpu_put(vcpu); + + kvm_reset_vcpu_core(vcpu); + + /* Reset special registers */ + vcpu_reset_hcr(vcpu); + + if (reset_state.reset) { + *vcpu_pc(vcpu) = reset_state.pc; + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); + vcpu_set_reg(vcpu, 0, reset_state.r0); + } + + /* Load new vx-regs into HW if they are currently loaded */ + if (current->thread.kfpu_flags) + load_vx_regs(vcpu->arch.ctxt.vregs); + + if (loaded) + vcpu_load(vcpu); + + preempt_enable(); +} + +int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) +{ + return -EINVAL; +}
diff --git a/arch/s390/kvm/arm64/trace.h b/arch/s390/kvm/arm64/trace.h new file mode 100644 index 0000000..ee232c4 --- /dev/null +++ b/arch/s390/kvm/arm64/trace.h
@@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_ARM64_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define KVM_ARM64_TRACE_KVM_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm_arm64 +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR kvm_arm64 + +TRACE_EVENT(kvm_entry, + TP_PROTO(unsigned long vcpu_pc), + TP_ARGS(vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + ), + + TP_fast_assign(__entry->vcpu_pc = vcpu_pc;), + + TP_printk("PC: 0x%016lx", __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_exit, + TP_PROTO(unsigned int esr_ec, unsigned long vcpu_pc), + TP_ARGS(esr_ec, vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned int, esr_ec ) + __field(unsigned long, vcpu_pc ) + ), + + TP_fast_assign( + __entry->esr_ec = esr_ec; + __entry->vcpu_pc = vcpu_pc; + ), + + TP_printk("HSR_EC: 0x%04x, PC: 0x%016lx", + __entry->esr_ec, + __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_mmio_nisv, + TP_PROTO(unsigned long vcpu_pc, unsigned long esr, + unsigned long far, unsigned long ipa), + TP_ARGS(vcpu_pc, esr, far, ipa), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_pc ) + __field( unsigned long, esr ) + __field( unsigned long, far ) + __field( unsigned long, ipa ) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->esr = esr; + __entry->far = far; + __entry->ipa = ipa; + ), + + TP_printk("ipa %#016lx, esr %#016lx, far %#016lx, pc %#016lx", + __entry->ipa, __entry->esr, + __entry->far, __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_guest_fault, + TP_PROTO(unsigned long vcpu_pc, unsigned long hsr, + unsigned long hxfar, + unsigned long long ipa), + TP_ARGS(vcpu_pc, hsr, hxfar, ipa), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_pc ) + __field( unsigned long, hsr ) + __field( unsigned long, hxfar ) + __field( unsigned long long, ipa ) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->hsr = hsr; + __entry->hxfar = hxfar; + __entry->ipa = ipa; + ), + + TP_printk("ipa %#llx, hsr %#08lx, hxfar %#08lx, pc %#016lx", + __entry->ipa, __entry->hsr, + __entry->hxfar, __entry->vcpu_pc) +); + +#endif /* KVM_ARM64_TRACE_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include <trace/define_trace.h>
diff --git a/arch/s390/kvm/gmap/faultin.c b/arch/s390/kvm/gmap/faultin.c index 1dc7980..2e93bd5 100644 --- a/arch/s390/kvm/gmap/faultin.c +++ b/arch/s390/kvm/gmap/faultin.c
@@ -10,6 +10,11 @@ #include "gmap.h" #include "faultin.h" +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else +#include "s390.h" +#endif bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); #define CREATE_TRACE_POINTS
diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 3f3fa86..bf91e3d 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c
@@ -21,7 +21,11 @@ #include "dat.h" #include "gmap.h" +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else #include "s390.h" +#endif #include "faultin.h" static int gmap_limit_to_type(gfn_t limit)
diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index c2ffb5e..4e2da60 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c
@@ -3,7 +3,11 @@ #include <linux/kvm_types.h> #include <linux/kvm_host.h> +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else #include "s390.h" +#endif #include "gmap.h" #include "dat.h" #include "kvm_mmu.h" @@ -76,7 +80,7 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, return -EINVAL; } - if (kvm->arch.migration_mode) { + if (kvm_s390_is_migration_mode(kvm)) { /* * Turn off migration mode when: * - userspace creates a new memslot with dirty logging off,
diff --git a/arch/s390/kvm/gmap/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h index 15ec88b..1dce3ef 100644 --- a/arch/s390/kvm/gmap/trace_gmap.h +++ b/arch/s390/kvm/gmap/trace_gmap.h
@@ -4,13 +4,27 @@ #include <linux/tracepoint.h> +#undef UNPACK_KVM +#define UNPACK_KVM(name) name #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM UNPACK_KVM(KVM_DEV_NAME) #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH ../gmap #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE trace_gmap +#ifdef KVM_S390_ARM64 +#define __KVM_FIELDS \ + __field(unsigned long, pstate) \ + __field(unsigned long, pc) +#define __KVM_ASSIGN ({\ + __entry->pstate = vcpu->arch.sae_block.pstate; \ + __entry->pc = vcpu->arch.sae_block.pc; \ + }) +#define __KVM_PRINT \ + __entry->pstate, \ + __entry->pc +#else #define __KVM_FIELDS \ __field(unsigned long, pswmask) \ __field(unsigned long, pswaddr) @@ -21,6 +35,7 @@ #define __KVM_PRINT \ __entry->pswmask,\ __entry->pswaddr +#endif TRACE_EVENT(kvm_s390_major_guest_pfault, TP_PROTO(struct kvm_vcpu *vcpu),
diff --git a/arch/s390/kvm/s390/Makefile b/arch/s390/kvm/s390/Makefile index 762a638..b91334d 100644 --- a/arch/s390/kvm/s390/Makefile +++ b/arch/s390/kvm/s390/Makefile
@@ -4,7 +4,7 @@ include $(srctree)/virt/kvm/Makefile.kvm include $(srctree)/arch/s390/kvm/gmap/Makefile -ccflags-y := -I$(src) -I$(srctree)/arch/s390/kvm/gmap +ccflags-y += -I$(src) -I$(srctree)/arch/s390/kvm/gmap kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o
diff --git a/arch/s390/kvm/s390/intercept.c b/arch/s390/kvm/s390/intercept.c index ca1205d..adfde2a 100644 --- a/arch/s390/kvm/s390/intercept.c +++ b/arch/s390/kvm/s390/intercept.c
@@ -11,6 +11,8 @@ #include <linux/kvm_host.h> #include <linux/errno.h> #include <linux/pagemap.h> +#include <linux/cleanup.h> +#include <linux/slab.h> #include <asm/asm-offsets.h> #include <asm/irq.h> @@ -411,9 +413,9 @@ static int handle_partial_execution(struct kvm_vcpu *vcpu) */ int handle_sthyi(struct kvm_vcpu *vcpu) { + struct sthyi_sctns *sctns __free(kfree) = NULL; int reg1, reg2, cc = 0, r = 0; u64 code, addr, rc = 0; - struct sthyi_sctns *sctns = NULL; if (!test_kvm_facility(vcpu->kvm, 74)) return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); @@ -430,37 +432,32 @@ int handle_sthyi(struct kvm_vcpu *vcpu) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); if (code & 0xffff) { - cc = 3; - rc = 4; - goto out; + vcpu->run->s.regs.gprs[reg2 + 1] = 4; + kvm_s390_set_psw_cc(vcpu, 3); + return 0; } if (!kvm_s390_pv_cpu_is_protected(vcpu) && (addr & ~PAGE_MASK)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); - sctns = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + sctns = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!sctns) return -ENOMEM; cc = sthyi_fill(sctns, &rc); - if (cc < 0) { - free_page((unsigned long)sctns); + if (cc < 0) return cc; - } -out: + if (!cc) { if (kvm_s390_pv_cpu_is_protected(vcpu)) { memcpy(sida_addr(vcpu->arch.sie_block), sctns, PAGE_SIZE); } else { r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE); - if (r) { - free_page((unsigned long)sctns); + if (r) return kvm_s390_inject_prog_cond(vcpu, r); - } } } - free_page((unsigned long)sctns); vcpu->run->s.regs.gprs[reg2 + 1] = rc; kvm_s390_set_psw_cc(vcpu, cc); return r; @@ -536,6 +533,15 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); spin_unlock_irqrestore(&fi->lock, flags); + + /* + * We missed the floating IRQ kick since we can only inject + * when we end up here and not when the irq was injected via + * the FLIC. + * + * Now that we have cleared the masking we can kick cpus. + */ + kvm_s390_pv_sclp_kick(vcpu); return 0; }
diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index f892f43..31a6074 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c
@@ -1918,49 +1918,126 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) return 0; } +static u64 inti_to_irq_pend_mask(u64 type, int isc) +{ + switch (type) { + case KVM_S390_MCHK: + /* Only repressible machine checks are floating */ + return BIT(IRQ_PEND_MCHK_REP); + case KVM_S390_INT_VIRTIO: + return BIT(IRQ_PEND_VIRTIO); + case KVM_S390_INT_SERVICE: + return BIT(IRQ_PEND_EXT_SERVICE) | + BIT(IRQ_PEND_EXT_SERVICE_EV); + case KVM_S390_INT_PFAULT_DONE: + return BIT(IRQ_PEND_PFAULT_DONE); + case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + return BIT(isc_to_irq_type(isc)); + default: + return 0; + } +} + /* - * Find a destination VCPU for a floating irq and kick it. + * Setup intervention masks to catch running vcpus that hopefully open + * their masks soonish and kick sleeping vcpus to motivate them to + * take IRQs. */ -static void __floating_irq_kick(struct kvm *kvm, u64 type) +static void vcpu_intervention_kick(struct kvm_vcpu *vcpu, u64 type) +{ + /* make the VCPU drop out of the SIE, or wake it up if sleeping */ + switch (type) { + case KVM_S390_MCHK: + kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); + break; + case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + if (!(type & KVM_S390_INT_IO_AI_MASK && + vcpu->kvm->arch.gisa_int.origin) || + kvm_s390_pv_cpu_get_handle(vcpu)) + kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT); + break; + default: + kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); + break; + } + kvm_s390_vcpu_wakeup(vcpu); +} + +static void kick_cpu_irq(struct kvm *kvm, u64 type, u64 parm) { struct kvm_vcpu *dst_vcpu; int sigcpu, online_vcpus, nr_tries = 0; + u64 irq_pend_mask; + unsigned long i; online_vcpus = atomic_read(&kvm->online_vcpus); if (!online_vcpus) return; + irq_pend_mask = inti_to_irq_pend_mask(type, parm); for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) { sigcpu %= online_vcpus; dst_vcpu = kvm_get_vcpu(kvm, sigcpu); - if (!is_vcpu_stopped(dst_vcpu)) + if (!is_vcpu_stopped(dst_vcpu) && + deliverable_irqs(dst_vcpu) & irq_pend_mask) break; /* avoid endless loops if all vcpus are stopped */ - if (nr_tries++ >= online_vcpus) - return; + if (nr_tries++ >= online_vcpus) { + dst_vcpu = NULL; + break; + } } - /* make the VCPU drop out of the SIE, or wake it up if sleeping */ - switch (type) { - case KVM_S390_MCHK: - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT); - break; - case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: - if (!(type & KVM_S390_INT_IO_AI_MASK && - kvm->arch.gisa_int.origin) || - kvm_s390_pv_cpu_get_handle(dst_vcpu)) - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); - break; - default: - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT); - break; + /* Nobody was enabled, time to wake all of them */ + if (!dst_vcpu) { + kvm_for_each_vcpu(i, dst_vcpu, kvm) + vcpu_intervention_kick(dst_vcpu, type); + return; } - kvm_s390_vcpu_wakeup(dst_vcpu); + + vcpu_intervention_kick(dst_vcpu, type); +} + +void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu) +{ + /* + * The cpu that called sclp likely will also take the IRQ, no + * need to kick anyone. + */ + if (deliverable_irqs(vcpu) & BIT(IRQ_PEND_EXT_SERVICE)) + return; + + /* + * For the other cases we might have sleeping cpus with open + * masks. Time to find and kick them. + */ + kick_cpu_irq(vcpu->kvm, KVM_S390_INT_SERVICE, -1); +} + +/* + * Find a destination VCPU for a floating irq and kick it. + */ +static void __floating_irq_kick(struct kvm *kvm, u64 type, u64 parm) +{ + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + + /* + * No need to kick on non-ev service IRQs for PV VMs, we're + * not allowed to inject anyway. We need to wait for the sclp + * instruction notification AFTER re-entry of the vcpu that + * handled the instruction intercept. + */ + if (type == KVM_S390_INT_SERVICE && !(parm & SCCB_EVENT_PENDING) && + test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs)) + return; + + kick_cpu_irq(kvm, type, parm); } static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { u64 type = READ_ONCE(inti->type); + u64 parm = -1; int rc; switch (type) { @@ -1971,12 +2048,15 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) rc = __inject_virtio(kvm, inti); break; case KVM_S390_INT_SERVICE: + parm = inti->ext.ext_params & SCCB_EVENT_PENDING; rc = __inject_service(kvm, inti); break; case KVM_S390_INT_PFAULT_DONE: rc = __inject_pfault_done(kvm, inti); break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + /* Grab isc here since __inject_io() might free inti */ + parm = int_word_to_isc(inti->io.io_int_word); rc = __inject_io(kvm, inti); break; default: @@ -1985,7 +2065,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) if (rc) return rc; - __floating_irq_kick(kvm, type); + __floating_irq_kick(kvm, type, parm); return 0; } @@ -3690,7 +3770,7 @@ void kvm_s390_gib_destroy(void) } chsc_sgib(0); unregister_adapter_interrupt(&gib_alert_irq); - free_page((unsigned long)gib); + kfree(gib); gib = NULL; } @@ -3704,7 +3784,7 @@ int __init kvm_s390_gib_init(u8 nisc) goto out; } - gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); + gib = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT | GFP_DMA); if (!gib) { rc = -ENOMEM; goto out; @@ -3723,7 +3803,7 @@ int __init kvm_s390_gib_init(u8 nisc) gib_origin = virt_to_phys(gib); if (chsc_sgib(gib_origin)) { pr_err("Associating the GIB with the AIV facility failed\n"); - free_page((unsigned long)gib); + kfree(gib); gib = NULL; rc = -EIO; goto out_unreg_gal; @@ -3743,7 +3823,7 @@ int __init kvm_s390_gib_init(u8 nisc) out_unreg_gal: unregister_adapter_interrupt(&gib_alert_irq); out_free_gib: - free_page((unsigned long)gib); + kfree(gib); gib = NULL; out: return rc;
diff --git a/arch/s390/kvm/s390/pci.c b/arch/s390/kvm/s390/pci.c index 82892e1..d79bd3b 100644 --- a/arch/s390/kvm/s390/pci.c +++ b/arch/s390/kvm/s390/pci.c
@@ -498,17 +498,22 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev) * available, enable them and let userspace indicate whether or not they will * be used (specify SHM bit to disable). */ -static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm) +static int kvm_s390_pci_register_kvm(void *opaque, struct file *kvm_file) { struct zpci_dev *zdev = opaque; + struct kvm *kvm; int rc; if (!zdev) return -EINVAL; + kvm = file_to_kvm_s390(kvm_file); + if (!kvm) + return -ENOENT; + mutex_lock(&zdev->kzdev_lock); - if (zdev->kzdev || zdev->gisa != 0 || !kvm) { + if (zdev->kzdev || zdev->gisa != 0) { mutex_unlock(&zdev->kzdev_lock); return -EINVAL; }
diff --git a/arch/s390/kvm/s390/priv.c b/arch/s390/kvm/s390/priv.c index b3cb2c2..856939a 100644 --- a/arch/s390/kvm/s390/priv.c +++ b/arch/s390/kvm/s390/priv.c
@@ -14,6 +14,7 @@ #include <linux/mm_types.h> #include <linux/pgtable.h> #include <linux/io.h> +#include <linux/slab.h> #include <asm/asm-offsets.h> #include <asm/facility.h> #include <asm/current.h> @@ -869,7 +870,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu) int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; - unsigned long mem = 0; + void *mem = NULL; u64 operand2; int rc = 0; u8 ar; @@ -911,19 +912,19 @@ static int handle_stsi(struct kvm_vcpu *vcpu) switch (fc) { case 1: /* same handling for 1 and 2 */ case 2: - mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); + mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!mem) goto out_no_data; - if (stsi((void *) mem, fc, sel1, sel2)) + if (stsi(mem, fc, sel1, sel2)) goto out_no_data; break; case 3: if (sel1 != 2 || sel2 != 2) goto out_no_data; - mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); + mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!mem) goto out_no_data; - handle_stsi_3_2_2(vcpu, (void *) mem); + handle_stsi_3_2_2(vcpu, mem); break; case 15: /* fc 15 is fully handled in userspace */ insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); @@ -931,10 +932,10 @@ static int handle_stsi(struct kvm_vcpu *vcpu) return -EREMOTE; } if (kvm_s390_pv_cpu_is_protected(vcpu)) { - memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); + memcpy(sida_addr(vcpu->arch.sie_block), mem, PAGE_SIZE); rc = 0; } else { - rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); + rc = write_guest(vcpu, operand2, ar, mem, PAGE_SIZE); } if (rc) { rc = kvm_s390_inject_prog_cond(vcpu, rc); @@ -945,14 +946,14 @@ static int handle_stsi(struct kvm_vcpu *vcpu) rc = -EREMOTE; } trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); - free_page(mem); + kfree(mem); kvm_s390_set_psw_cc(vcpu, 0); vcpu->run->s.regs.gprs[0] = 0; return rc; out_no_data: kvm_s390_set_psw_cc(vcpu, 3); out: - free_page(mem); + kfree(mem); return rc; }
diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index e4912db..18fb200 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c
@@ -3273,8 +3273,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) goto out_err; BUILD_BUG_ON(sizeof(struct sie_page2) != 4096); - kvm->arch.sie_page2 = - (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); + kvm->arch.sie_page2 = kzalloc_obj(*kvm->arch.sie_page2, + GFP_KERNEL_ACCOUNT | GFP_DMA); if (!kvm->arch.sie_page2) goto out_err; @@ -3369,7 +3369,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return 0; out_err: - free_page((unsigned long)kvm->arch.sie_page2); + kfree(kvm->arch.sie_page2); debug_unregister(kvm->arch.dbf); sca_dispose(kvm); KVM_EVENT(3, "creation of vm failed: %d", rc); @@ -3427,7 +3427,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm); debug_unregister(kvm->arch.dbf); - free_page((unsigned long)kvm->arch.sie_page2); + kfree(kvm->arch.sie_page2); kvm_s390_destroy_adapters(kvm); kvm_s390_clear_float_irqs(kvm); kvm_s390_vsie_destroy(kvm); @@ -3656,13 +3656,13 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) { if (vcpu->arch.sie_block->cbrlo) - free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); + kfree(phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK)); vcpu->arch.sie_block->cbrlo = 0; } int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) { - void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + void *cbrlo_page = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!cbrlo_page) return -ENOMEM; @@ -5780,6 +5780,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, { } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + + return 0; +} + /** * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables * @vcpu: the vcpu that shall appear to have generated the fault-in. @@ -5806,9 +5814,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_ gpa_t end; int rc; - if (kvm_is_ucontrol(vcpu->kvm)) - return -EINVAL; - rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); if (rc == PGM_ADDRESSING) return -ENOENT;
diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h index aa0d1d0..9ed1af5 100644 --- a/arch/s390/kvm/s390/s390.h +++ b/arch/s390/kvm/s390/s390.h
@@ -375,6 +375,7 @@ enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer); int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); +void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu); int __must_check kvm_s390_inject_vm(struct kvm *kvm, struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti);
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 51db92c..56ec498 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c
@@ -828,6 +828,14 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, return 0; } +static void emit_shift_count32(struct bpf_jit *jit, u32 src_reg) +{ + /* risbg %w1,%src,59,128+63,0 */ + _EMIT6(0xec000000 | reg(REG_W1, src_reg) << 16 | 59 << 8 | (0x80 | 63), 0x0055); + REG_SET_SEEN(REG_W1); + REG_SET_SEEN(src_reg); +} + static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) { struct bpf_jit_probe probe; @@ -1400,9 +1408,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_LSH */ - case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ - /* sll %dst,0(%src) */ - EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << ((u32) src & 31) */ + emit_shift_count32(jit, src_reg); + /* sll %dst,0(%w1) */ + EMIT4_DISP(0x89000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ @@ -1425,9 +1434,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_RSH */ - case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ - /* srl %dst,0(%src) */ - EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> ((u32) src & 31) */ + emit_shift_count32(jit, src_reg); + /* srl %dst,0(%w1) */ + EMIT4_DISP(0x88000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ @@ -1450,9 +1460,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_ARSH */ - case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ - /* sra %dst,%dst,0(%src) */ - EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= (u32) src & 31 */ + emit_shift_count32(jit, src_reg); + /* sra %dst,0(%w1) */ + EMIT4_DISP(0x8a000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ @@ -2260,10 +2271,11 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, else jit->frame_off = 0; - kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); - if (kern_arena) + if (bpf_prog_uses_arena_insns(fp)) { + kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); jit->kern_arena = _EMIT_CONST_U64(kern_arena); - jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + } bpf_jit_prologue(jit, fp); if (bpf_set_addr(jit, 0) < 0)
diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index ba1e42d..e72b48e 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile
@@ -3,14 +3,18 @@ # Makefile for s390 specific build tools # +include $(srctree)/arch/s390/tools/Makefile.arm64h + kapi := arch/$(ARCH)/include/generated/asm kapi-hdrs-y := $(kapi)/dis-defs.h kapi-hdrs-y += $(kapi)/facility-defs.h kapi-hdrs-y += $(kapi)/pgmcheck-defs.h +kapi-hdrs-$(CONFIG_KVM) += $(ARM64_TARGETS) + PHONY += kapi -kapi: $(kapi-hdrs-y) +kapi: $(kapi-hdrs-y) $(kapi-hdrs-m) hostprogs += gen_bitmaps hostprogs += gen_opcode_table
diff --git a/arch/s390/tools/Makefile.arm64h b/arch/s390/tools/Makefile.arm64h new file mode 100644 index 0000000..f3f8e9d --- /dev/null +++ b/arch/s390/tools/Makefile.arm64h
@@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Share ARM64 headers with s390 KVM +# + +arm64api := $(objtree)/arch/$(ARCH)/include/generated/arm64 +arm64uapi := $(objtree)/arch/$(ARCH)/include/generated/uapi/arm64 + +# List of ARM64 headers to share with s390 KVM +# Format: [uapi:]header.h or header-gen.h +# +# * header.h - from arch/arm64/include/asm/ to arm64/ +# * uapi:header.h - from arch/arm64/include/uapi/asm/ to uapi/arm64/ +# * header-gen.h - generated as header-gen.h from ARM64 header header.h +# If a header contains partial markers (#if ARM64_S390_COMMON == 1) only this part will be copied. +# If the file does not contain any markers the whole file will be copied. +ARM64_SHARED_HEADERS := \ + brk-imm.h \ + esr.h \ + kvm_arm.h \ + kvm_emulate-gen.h \ + kvm_host.h \ + ptrace-gen.h \ + sysreg-gen.h \ + uapi:hwcap.h \ + uapi:kvm.h \ + uapi:ptrace.h \ + uapi:sve_context.h + +filechk_gen_arm_hdr = \ + $(AWK) -f $(src)/copy-arm64h.awk $< || \ + $(AWK) -f $(src)/copy-arm64h-full.awk $< || \ + { echo "Error: Unbalanced ARM64_S390_COMMON markers in $<" >&2; exit 1; } + +arm64-type = $(firstword $(subst :, ,$(1))) +arm64-base = $(or $(word 2,$(subst :, ,$(1))),$(1)) +arm64-src-base = $(patsubst %-gen.h,%.h,$(call arm64-base,$(1))) + +arm64-src = $(srctree)/arch/arm64/include/$(if $(filter uapi,$(call arm64-type,$(1))),uapi/)asm/$(call arm64-src-base,$(1)) +arm64-dst = $(if $(filter uapi,$(call arm64-type,$(1))),$(arm64uapi),$(arm64api))/$(call arm64-base,$(1)) + +$(foreach hdr,$(ARM64_SHARED_HEADERS),\ + $(eval $(call arm64-dst,$(hdr)): $(call arm64-src,$(hdr)) $(src)/copy-arm64h.awk $(src)/copy-arm64h-full.awk FORCE)) + +ARM64_TARGETS := $(sort $(foreach hdr,$(ARM64_SHARED_HEADERS),$(call arm64-dst,$(hdr)))) +$(ARM64_TARGETS): + $(call filechk,gen_arm_hdr) + +filechk_gen_sysreg_arm64 = \ + $(AWK) -f $(srctree)/arch/arm64/tools/gen-sysreg.awk $(srctree)/arch/arm64/tools/sysreg + +$(arm64api)/sysreg-defs.h: $(srctree)/arch/arm64/tools/gen-sysreg.awk $(srctree)/arch/arm64/tools/sysreg FORCE + $(call filechk,gen_sysreg_arm64) + +ARM64_TARGETS += $(arm64api)/sysreg-defs.h
diff --git a/arch/s390/tools/copy-arm64h-full.awk b/arch/s390/tools/copy-arm64h-full.awk new file mode 100644 index 0000000..ad7bb98 --- /dev/null +++ b/arch/s390/tools/copy-arm64h-full.awk
@@ -0,0 +1,50 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Process entire ARM64 headers for sharing with s390 KVM +# +# Usage: copy-arm64h-full.awk <input_file> +# +# This processes the entire file (unlike copy-arm64h.awk which only extracts marked sections) + +BEGIN { + max_guard_line = 25 + guard_found = 0 + header_added = 0 +} + +# Find and process the include guard in the first few lines +NR <= max_guard_line && !guard_found && /^#ifndef [A-Z_0-9]+$/ { + guard_name = $2 + guard_found = 1 + print "/* This header was copied from " FILENAME " */" + print "" + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " NR " \"" FILENAME "\"" + print + header_added = 1 + next +} + +# Transform include directives +/^#include <uapi\/asm\// { + sub(/<uapi\/asm\//, "<uapi/arm64/") + print + next +} + +/^#include <asm\// { + sub(/<asm\//, "<arm64/") + print + next +} + +/ARM64_S390_COMMON/ { + exit 1 +} + +# Pass through all other lines +{ + print +}
diff --git a/arch/s390/tools/copy-arm64h.awk b/arch/s390/tools/copy-arm64h.awk new file mode 100644 index 0000000..9a338d7 --- /dev/null +++ b/arch/s390/tools/copy-arm64h.awk
@@ -0,0 +1,83 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Extract marked sections from ARM64 headers for sharing with s390 KVM +# +# Usage: copy-arm64h.awk <input_file> +# +# Extracts all sections between start/end markers. If no markers found, signals for fallback. +BEGIN { + # Constants + start_pattern = "^#if ARM64_S390_COMMON == 1$" + end_pattern = "^#endif \\/\\* ARM64_S390_COMMON == 1 \\*\\/$" + max_guard_line = 25 + + # State variables + copying = found_marker = error = 0 + guard_name = "" + file_header_done = 0 + depth = 0 +} + +NR <= max_guard_line && !guard_name && /^#ifndef [A-Za-z0-9_]+$/ { + guard_name = $2 + next +} + +NR > max_guard_line && !guard_name && !file_header_done { + print "error: no include guard found in first " max_guard_line " lines" > "/dev/stderr" + file_header_done = 1 + error = 1; exit +} + +$0 ~ start_pattern { + if (copying) { error = 1; exit } + if (!found_marker) { + print "/*" + print " * This header was automatically generated from " FILENAME + print " * Do not modify this file directly." + print " */" + print "#ifndef " guard_name + print "#define " guard_name + print "" + } + depth = copying= found_marker = 1 + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " (NR + 1) " \"" FILENAME "\"" + next +} + +guard_name && !found_marker { + next +} + +$0 ~ /^[ \t]*#[ \t]*endif/ { + depth-- + if (depth == 0) { + if ($0 ~ end_pattern ) { + copying = 0 + next + } + error = 1; exit + } +} + +/ARM64_S390_COMMON/ { error = 1; exit } + +copying { + if ($0 ~ /^[ \t]*#[ \t]*if(def|ndef)?([ \t]|$)/) depth++ + gsub(/#include <asm\//, "#include <arm64/") + gsub(/#include <uapi\/asm\//, "#include <uapi/arm64/") + print + next +} + +END { + if (error) exit 1 + if (found_marker) { + print "" + print "#endif /* " guard_name " */" + } + exit !found_marker || copying +}
diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index def2659..0e4773c 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt
@@ -594,6 +594,9 @@ b9a1 tpei RRE_RR b9a2 ptf RRE_R0 b9a4 uvc RRF_URR +b9a5 sae RRE_R0 +b9a6 lasrm RRE_R0 +b9a7 stiasrm RRE_R0 b9aa lptea RRF_RURR2 b9ab essa RRF_U0RR b9ac irbm RRE_RR
diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 30899dd..881c340 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c
@@ -20,6 +20,7 @@ * * Based on Virtio MMIO driver by Pawel Moll, copyright 2011-2014, ARM Ltd. */ +#include <linux/interrupt.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> @@ -869,6 +870,14 @@ static void vu_reset(struct virtio_device *vdev) vu_dev->status = 0; } +static void vu_synchronize_cbs(struct virtio_device *vdev) +{ + struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); + + if (vu_dev->irq != UM_IRQ_ALLOC) + synchronize_irq(vu_dev->irq); +} + static void vu_del_vq(struct virtqueue *vq) { struct virtio_uml_vq_info *info = vq->priv; @@ -1121,6 +1130,7 @@ static const struct virtio_config_ops virtio_uml_config_ops = { .reset = vu_reset, .find_vqs = vu_find_vqs, .del_vqs = vu_del_vqs, + .synchronize_cbs = vu_synchronize_cbs, .get_features = vu_get_features, .finalize_features = vu_finalize_features, .bus_name = vu_bus_name,
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a02d953..a5e29ff 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig
@@ -234,6 +234,7 @@ select HAVE_SAMPLE_FTRACE_DIRECT if X86_64 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI if X86_64 select HAVE_EBPF_JIT + select HAVE_EBPF_JIT_KASAN if X86_64 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 9900a5c..b90e6f4 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c
@@ -579,6 +579,10 @@ void __init hyperv_init(void) old_setup_percpu_clockev = x86_init.timers.setup_percpu_clockev; x86_init.timers.setup_percpu_clockev = hv_stimer_setup_percpu_clockev; +#ifdef CONFIG_HYPERV_PVIOMMU + x86_init.iommu.iommu_init = hv_iommu_init; +#endif + hv_apic_init(); x86_init.pci.arch_init = hv_pci_init;
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f629447..61930f0 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h
@@ -238,6 +238,7 @@ // free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ +#define X86_FEATURE_PML ( 8*32+23) /* AMD Page Modification logging */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* "fsgsbase" RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/
diff --git a/arch/x86/include/asm/crash_reserve.h b/arch/x86/include/asm/crash_reserve.h index 7835b2c..047f638 100644 --- a/arch/x86/include/asm/crash_reserve.h +++ b/arch/x86/include/asm/crash_reserve.h
@@ -18,7 +18,7 @@ * no good way to detect the paging mode of the target kernel which will be * loaded for dumping. */ -extern unsigned long swiotlb_size_or_default(void); +extern unsigned long swiotlb_default_pool_size(void); #ifdef CONFIG_X86_32 # define CRASH_ADDR_LOW_MAX SZ_512M @@ -33,7 +33,7 @@ extern unsigned long swiotlb_size_or_default(void); static inline unsigned long crash_low_size_default(void) { #ifdef CONFIG_X86_64 - return max(swiotlb_size_or_default() + (8UL << 20), 256UL << 20); + return max(swiotlb_default_pool_size() + (8UL << 20), 256UL << 20); #else return 0; #endif
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h index 3be2451..4e1e55c 100644 --- a/arch/x86/include/asm/iommu.h +++ b/arch/x86/include/asm/iommu.h
@@ -4,6 +4,11 @@ #include <linux/acpi.h> +#ifdef CONFIG_IOMMU_API +#include <linux/iommu.h> +#include <linux/sizes.h> +#endif + #include <asm/e820/api.h> extern int force_iommu, no_iommu; @@ -21,6 +26,21 @@ extern bool x86_swiotlb_enable; /* 10 seconds */ #define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) +#ifdef CONFIG_IOMMU_API +static inline int iommu_add_resv_x86_msi_region(struct list_head *head) +{ + struct iommu_resv_region *region; + + region = iommu_alloc_resv_region(0xfee00000, SZ_1M, 0, + IOMMU_RESV_MSI, GFP_KERNEL); + if (!region) + return -ENOMEM; + + list_add_tail(®ion->list, head); + return 0; +} +#endif + static inline int __init arch_rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr) {
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 5c358c4..4dde4e3e 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -82,6 +82,7 @@ KVM_X86_OP(interrupt_allowed) KVM_X86_OP(nmi_allowed) KVM_X86_OP(get_nmi_mask) KVM_X86_OP(set_nmi_mask) +KVM_X86_OP_OPTIONAL(clear_hlt) KVM_X86_OP(enable_nmi_window) KVM_X86_OP(enable_irq_window) KVM_X86_OP_OPTIONAL(update_cr8_intercept) @@ -151,7 +152,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT KVM_X86_OP_OPTIONAL_RET0(gmem_make_private) #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) KVM_X86_OP_OPTIONAL(gmem_make_shared) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 20b9db3..1ee4b12 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h
@@ -16,6 +16,7 @@ #include <linux/irq_work.h> #include <linux/irq.h> #include <linux/workqueue.h> +#include <linux/xarray.h> #include <linux/kvm.h> #include <linux/kvm_para.h> @@ -44,7 +45,12 @@ #include <hyperv/hvhdk.h> +#define kvm_file_to_kvm_arch x86 + #define __KVM_HAVE_ARCH_VCPU_DEBUGFS +#define __KVM_HAVE_MSI +#define __KVM_HAVE_NO_POLL +#define __KVM_HAVE_READONLY_MEM /* * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if @@ -779,6 +785,8 @@ struct kvm_vcpu_arch { */ struct kvm_mmu_memory_cache mmu_external_spt_cache; + struct page *pml_page; + /* * QEMU userspace and the guest each have their own FPU state. * In vcpu_run, we switch between the user and guest FPU contexts. @@ -855,7 +863,7 @@ struct kvm_vcpu_arch { gpa_t time; s8 pvclock_tsc_shift; u32 pvclock_tsc_mul; - unsigned int hw_tsc_khz; + u64 hw_tsc_hz; struct gfn_to_pfn_cache pv_time; /* set guest stopped flag in pvclock flags field */ bool pvclock_set_guest_stopped_request; @@ -964,7 +972,7 @@ struct kvm_vcpu_arch { bool pv_unhalted; } pv; - int pending_ioapic_eoi; + u8 pending_ioapic_eoi; int pending_external_vector; int highest_stale_pending_ioapic_eoi; @@ -1104,7 +1112,7 @@ struct kvm_xen { bool runstate_update_flag; u8 upcall_vector; struct gfn_to_pfn_cache shinfo_cache; - struct idr evtchn_ports; + struct xarray evtchn_ports; unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; struct kvm_xen_hvm_config hvm_config; @@ -1156,6 +1164,7 @@ struct kvm_arch { u8 mmu_valid_gen; u8 vm_type; bool has_private_mem; + bool has_protected_page_tables; bool has_protected_state; bool has_protected_eoi; bool has_protected_pmu; @@ -1227,6 +1236,7 @@ struct kvm_arch { u64 last_tsc_write; u32 last_tsc_khz; u64 last_tsc_offset; + u64 last_tsc_scaling_ratio; u64 cur_tsc_nsec; u64 cur_tsc_write; u64 cur_tsc_offset; @@ -1242,6 +1252,9 @@ struct kvm_arch { u64 master_kernel_ns; u64 master_cycle_now; struct ratelimit_state kvmclock_update_rs; + u64 master_tsc_scaling_ratio; + s8 master_tsc_shift; + u32 master_tsc_mul; #ifdef CONFIG_KVM_HYPERV struct kvm_hv hyperv; @@ -1397,6 +1410,7 @@ struct kvm_arch { struct kvm_mmu_memory_cache split_desc_cache; gfn_t gfn_direct_bits; + int mirror_root_level; /* * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero @@ -1569,7 +1583,7 @@ struct kvm_x86_ops { * Can potentially get non-canonical addresses through INVLPGs, which * the implementation may choose to ignore if appropriate. */ - void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); + void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); /* * Flush any TLB entries created by the guest. Like tlb_flush_gva(), @@ -1598,6 +1612,13 @@ struct kvm_x86_ops { int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); + /* + * Force the vCPU out of any hardware-tracked halted/inactive state so + * that it will fetch and execute instructions on the next VM-Enter. + * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across + * VM-Exit/VM-Enter; SVM has no equivalent VMCB field. + */ + void (*clear_hlt)(struct kvm_vcpu *vcpu); /* Whether or not a virtual NMI is pending in hardware. */ bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); /* @@ -1661,7 +1682,7 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable); void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); @@ -1724,7 +1745,7 @@ struct kvm_x86_ops { int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE @@ -1843,9 +1864,14 @@ enum kvm_intr_type { ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#if defined(CONFIG_KVM_SW_PROTECTED_VM) || \ + defined(CONFIG_KVM_INTEL_TDX) || \ + defined(CONFIG_KVM_AMD_SEV) #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private) +#endif #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)
diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index 3d04074..ea885dd 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h
@@ -44,13 +44,15 @@ struct kvm_page_track_notifier_node { struct kvm_page_track_notifier_node *node); }; -int kvm_page_track_register_notifier(struct kvm *kvm, +struct file; + +int kvm_page_track_register_notifier(struct file *file, struct kvm_page_track_notifier_node *n); -void kvm_page_track_unregister_notifier(struct kvm *kvm, +void kvm_page_track_unregister_notifier(struct file *file, struct kvm_page_track_notifier_node *n); -int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn); -int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn); +int kvm_write_track_add_gfn(struct file *file, gfn_t gfn); +int kvm_write_track_remove_gfn(struct file *file, gfn_t gfn); #else /* * Allow defining a node in a structure even if page tracking is disabled, e.g.
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index f64393e..20d947c 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h
@@ -313,6 +313,10 @@ static inline void mshv_vtl_return_hypercall(void) {} static inline void __mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0) {} #endif +#ifdef CONFIG_HYPERV_PVIOMMU +int __init hv_iommu_init(void); +#endif + #include <asm-generic/mshyperv.h> #endif
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index aa63431..83fd49b 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h
@@ -165,7 +165,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u8 reserved_9[22]; u64 allowed_sev_features; /* Offset 0x138 */ u64 guest_sev_features; /* Offset 0x140 */ - u8 reserved_10[664]; + u8 reserved_10[128]; + u64 pml_addr; /* Offset 0x1c8 */ + u16 pml_index; /* Offset 0x1d0 */ + u8 reserved_11[526]; /* * Offset 0x3e0, 32 bytes reserved * for use by hypervisor/software. @@ -240,10 +243,11 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) -#define SVM_MISC_ENABLE_NP BIT(0) -#define SVM_MISC_ENABLE_SEV BIT(1) -#define SVM_MISC_ENABLE_SEV_ES BIT(2) -#define SVM_MISC_ENABLE_GMET BIT(3) +#define SVM_MISC_ENABLE_NP BIT_ULL(0) +#define SVM_MISC_ENABLE_SEV BIT_ULL(1) +#define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) +#define SVM_MISC_ENABLE_GMET BIT_ULL(3) +#define SVM_MISC_ENABLE_PML BIT_ULL(11) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) @@ -637,8 +641,8 @@ static inline void __unused_size_checks(void) #define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT) #define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT) -#define SVM_EVTINJ_VALID (1 << 31) -#define SVM_EVTINJ_VALID_ERR (1 << 11) +#define SVM_EVTINJ_VALID BIT(31) +#define SVM_EVTINJ_VALID_ERR BIT(11) #define SVM_EVTINJ_RESERVED_BITS ~(SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | \ SVM_EVTINJ_VALID_ERR | SVM_EVTINJ_VALID)
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3f1b309..e49b836 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h
@@ -20,6 +20,14 @@ #include <asm/trapnr.h> #include <asm/vmxfeatures.h> +/* + * UNDEFINED is a KVM-defined value used to indicate that the basic exit reason + * is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter. + * The value itself is arbitrary, it just needs to not collide with a real exit + * reason. + */ +#define EXIT_REASON_UNDEFINED 0xdead + struct vmcs_hdr { u32 revision_id:31; u32 shadow_vmcs:1;
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 1585ec8..00ce265 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h
@@ -45,9 +45,7 @@ #define __KVM_HAVE_PIT #define __KVM_HAVE_IOAPIC #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_MSI #define __KVM_HAVE_USER_NMI -#define __KVM_HAVE_MSIX #define __KVM_HAVE_MCE #define __KVM_HAVE_PIT_STATE2 #define __KVM_HAVE_XEN_HVM
diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 010a45c..e806761 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h
@@ -101,6 +101,7 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_VMGEXIT 0x403 +#define SVM_EXIT_PML_FULL 0x407 /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001ull @@ -236,6 +237,7 @@ { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ { SVM_EXIT_VMGEXIT, "vmgexit" }, \ + { SVM_EXIT_PML_FULL, "pml_full" }, \ { SVM_VMGEXIT_MMIO_READ, "vmgexit_mmio_read" }, \ { SVM_VMGEXIT_MMIO_WRITE, "vmgexit_mmio_write" }, \ { SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \
diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 580b252..d867ab9 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c
@@ -55,6 +55,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 }, { X86_FEATURE_CPPC_PERF_PRIO, CPUID_EDX, 16, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_PML, CPUID_ECX, 4, 0x8000000a, 0 }, { X86_FEATURE_X2AVIC_EXT, CPUID_ECX, 6, 0x8000000a, 0 }, { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 },
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 538ed1e..1719059 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig
@@ -28,18 +28,14 @@ select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_READONLY_MEM select VHOST_TASK select KVM_ASYNC_PF select USER_RETURN_NOTIFIER - select KVM_MMIO select SCHED_INFO select PERF_EVENTS select GUEST_PERF_EVENTS select PERF_GUEST_MEDIATED_PMU - select HAVE_KVM_MSI select HAVE_KVM_CPU_RELAX_INTERCEPT - select HAVE_KVM_NO_POLL select VIRT_XFER_TO_GUEST_WORK select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_VFIO @@ -80,11 +76,24 @@ If in doubt, say "N". +config KVM_PRIVATE_MEMATTR_VM + bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)" + depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV + select KVM_VM_MEMORY_ATTRIBUTES + default KVM_INTEL_TDX || KVM_AMD_SEV + help + Enable support for tracking PRIVATE vs. SHARED memory using per-VM + memory attributes. Using per-VM attributes is deprecated in favor of + tracking PRIVATE state in guest_memfd. Select this if you need to run + CoCo VMs using a VMM that doesn't support guest_memfd memory + attributes. + + If unsure, say Y. + config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT depends on KVM_X86 && X86_64 - select KVM_GENERIC_MEMORY_ATTRIBUTES help Enable support for KVM software-protected VMs. Currently, software- protected VMs are purely a development and testing vehicle for @@ -135,7 +144,6 @@ bool "Intel Trust Domain Extensions (TDX) support" default y depends on INTEL_TDX_HOST - select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_POPULATE help Provides support for launching Intel Trust Domain Extensions (TDX) @@ -159,7 +167,6 @@ depends on KVM_AMD && X86_64 depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM - select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 0474604..c6bf463 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile
@@ -61,9 +61,7 @@ -e kvm_page_track_unregister_notifier \ -e kvm_write_track_add_gfn \ -e kvm_write_track_remove_gfn \ - -e kvm_get_kvm \ - -e kvm_get_kvm_safe \ - -e kvm_put_kvm + -e kvm_file_to_kvm_fn # Force grep to emit a goofy group separator that can in turn be replaced with # the above newline macro (newlines in Make are a nightmare). Note, grep only
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 851f151..adafd6f 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c
@@ -16,6 +16,7 @@ #include <linux/vmalloc.h> #include <linux/uaccess.h> #include <linux/sched/stat.h> +#include <linux/units.h> #include <asm/processor.h> #include <asm/user.h> @@ -483,20 +484,6 @@ int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu) return 36; } -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu) -{ - struct kvm_cpuid_entry2 *best; - - best = kvm_find_cpuid_entry(vcpu, 0x80000000); - if (!best || best->eax < 0x80000008) - goto not_found; - best = kvm_find_cpuid_entry(vcpu, 0x80000008); - if (best) - return (best->eax >> 16) & 0xff; -not_found: - return 0; -} - /* * This "raw" version returns the reserved GPA bits without any adjustments for * encryption technologies that usurp bits. The raw mask should be used if and @@ -1095,7 +1082,7 @@ void kvm_initialize_cpu_caps(void) ); kvm_cpu_cap_init(CPUID_24_1_ECX, - F(AVX10_VNNI_INT), + F(AVX10_V1_AUX), ); kvm_cpu_cap_init(CPUID_8000_0001_ECX, @@ -1171,6 +1158,10 @@ void kvm_initialize_cpu_caps(void) F(AMD_STIBP), F(AMD_STIBP_ALWAYS_ON), F(AMD_IBRS_SAME_MODE), + /* + * Vendor code also sets EFER_LMSLE_MBZ if KVM itself + * can't support EFER.LMSLE, e.g. if nested SVM is disabled. + */ PASSTHROUGH_F(EFER_LMSLE_MBZ), F(AMD_PSFD), F(AMD_IBPB_RET), @@ -1415,6 +1406,22 @@ static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 ind if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) entry->ecx = feature_bit(RDPID); return 1; + case 0x80000008: + /* + * Honor the guest's EFER_LMSLE_MBZ even if the underlying CPU + * allows setting EFER.LMSLE, e.g. to allow migrating a vCPU + * between hosts with and without EFER.LMSLE support. To avoid + * breaking existing setups that reflect KVM's supported CPUID + * into the guest, KVM doesn't advertise EFER_LMSLE_MBZ unless + * KVM *can't* support EFER.LMSLE=1. + */ + if (include_partially_emulated && + !kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) { + entry->ebx |= feature_bit(EFER_LMSLE_MBZ); + return 1; + } + /* Nothing in 0x80000008 is fully emulated, don't emit an entry. */ + return 0; default: return 0; }
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 8d863f4..46bfe86 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h
@@ -68,7 +68,6 @@ void __init kvm_init_xstate_sizes(void); u32 xstate_required_size(u64 xstate_bv, bool compacted); int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu); u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 604651cb..5d2e361 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c
@@ -802,7 +802,9 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, if (no_retry) return 0; + hv_hdr.message_flags.asu8 = 0; hv_hdr.message_flags.msg_pending = 1; + r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_flags, msg_off + @@ -1995,7 +1997,6 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; - bool full = false; if (!tdp_enabled) return -EINVAL; @@ -2006,7 +2007,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); - for (i = 0; i < count && !full; i++) { + for (i = 0; i < count; i++) { if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; @@ -2015,11 +2016,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) * pages to flush. */ gva = entries[i] & PAGE_MASK; - for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { + for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) continue; - kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); + kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); } ++vcpu->stat.tlb_flush;
diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 3e375af..10886be 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h
@@ -354,8 +354,8 @@ struct x86_emulate_ctxt { bool have_exception; struct x86_exception exception; - /* GPA available */ - bool gpa_available; + /* Saved GPA and permitted emulated accesses. */ + u64 gpa_access; gpa_t gpa_val; /*
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e1f3cea..ecf6118 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c
@@ -1552,7 +1552,7 @@ static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); } -static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) +static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, u8 vector) { int __maybe_unused trigger_mode; @@ -1622,7 +1622,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) * this interface assumes a trap-like exit, which has already finished * desired side effect including vISR and vPPR update. */ -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector) { struct kvm_lapic *apic = vcpu->arch.apic;
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index bd1098c..db071d0 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h
@@ -177,7 +177,7 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector); int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 46c033d..dc2aef7 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c
@@ -868,11 +868,12 @@ void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, list_del_init(&sp->possible_nx_huge_page_link); } -static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp) +static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, + enum kvm_mmu_type mmu_type) { sp->nx_huge_page_disallowed = false; - untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); + untrack_possible_nx_huge_page(kvm, sp, mmu_type); } static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, @@ -899,6 +900,7 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu */ #define KVM_RMAP_MANY BIT(0) +#ifndef CONFIG_PREEMPT_RT /* * rmaps and PTE lists are mostly protected by mmu_lock (the shadow MMU always * operates with mmu_lock held for write), but rmaps can be walked without @@ -1012,7 +1014,8 @@ static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) * actual locking is the same, but the caller is disallowed from modifying the * rmap, and so the unlock flow is a nop if the rmap is/was empty. */ -static unsigned long kvm_rmap_lock_readonly(struct kvm_rmap_head *rmap_head) +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) { unsigned long rmap_val; @@ -1036,6 +1039,35 @@ static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, __kvm_rmap_unlock(rmap_head, old_val); preempt_enable(); } +#else +static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) +{ + return atomic_long_read(&rmap_head->val); +} +static unsigned long kvm_rmap_lock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_write(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head, + unsigned long new_val) +{ + atomic_long_set_release(&rmap_head->val, new_val); +} + +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_read(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, + unsigned long old_val) { } +#endif /* * Returns the number of pointers in the rmap chain, not counting the new one. @@ -1749,11 +1781,24 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, gfn_t gfn; int level; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING)); + + /* + * For realtime kernels, do aging under mmu_lock (per-rmap locking is + * compiled out), as realtime deployments are unlikely to benefit from + * increased aging throughput and reduced jitter for memory-overcommitted + * nested VMs, whereas keeping preemption enabled is extremely valuable + * (mmu_lock becomes a sleepable lock on realtime kernels). + */ +#ifdef CONFIG_PREEMPT_RT + guard(read_lock)(&kvm->mmu_lock); +#endif + for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { for (gfn = range->start; gfn < range->end; gfn += KVM_PAGES_PER_HPAGE(level)) { rmap_head = gfn_to_rmap(gfn, level, range->slot); - rmap_val = kvm_rmap_lock_readonly(rmap_head); + rmap_val = kvm_rmap_lock_readonly(kvm, rmap_head); for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) { if (!is_accessed_spte(old_spte)) @@ -2795,7 +2840,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, } if (sp->nx_huge_page_disallowed) - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); sp->role.invalid = 1; @@ -3031,6 +3076,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, if (prefetch) return -EEXIST; + if (KVM_BUG_ON(sp->role.level != PG_LEVEL_4K, kvm)) + continue; + /* * TDP MMU page faults require an additional spinlock as they * run with mmu_lock held for read, not write, and the unsync @@ -3054,7 +3102,6 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, continue; } - WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K); kvm_unsync_page(kvm, sp); } if (locked) @@ -3431,7 +3478,7 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, is_private = fault->is_private; } else { max_level = PG_LEVEL_NUM; - is_private = kvm_mem_is_private(kvm, gfn); + is_private = kvm_is_private_gfn(kvm, gfn); } max_level = min(max_level, max_huge_page_level); @@ -3541,8 +3588,8 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access, base_gfn, fault->pfn, fault); @@ -3580,6 +3627,7 @@ static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa return RET_PF_RETRY; } + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } @@ -3644,13 +3692,13 @@ static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault * guest spinning on a #PF indefinitely, so don't attempt the fast path * in this case. * - * Note that the kvm_mem_is_private() check might race with an + * Note that the kvm_is_private_gfn() check might race with an * attribute update, but this will either result in the guest spinning * on RET_PF_SPURIOUS until the update completes, or an actual spurious * case might go down the slow path. Either case will resolve itself. */ if (kvm->arch.has_private_mem && - fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) + fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) return false; /* @@ -4638,7 +4686,7 @@ static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu, } r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn, - &fault->refcounted_page, &max_order); + &max_order); if (r) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return r; @@ -4718,7 +4766,7 @@ static int kvm_mmu_faultin_pfn(struct kvm_vcpu *vcpu, * Now that we have a snapshot of mmu_invalidate_seq we can check for a * private vs. shared mismatch. */ - if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) { + if (fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } @@ -4905,8 +4953,8 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, #ifndef CONFIG_X86_64 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ - if (WARN_ON_ONCE(fault_address >> 32)) - return -EFAULT; + if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm)) + return -EIO; #endif /* * Legacy #PF exception only have a 32-bit error code. Simply drop the @@ -5071,6 +5119,10 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + cond_resched(); r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level); } while (r == RET_PF_RETRY); @@ -5096,6 +5148,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, } } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!vcpu->kvm->arch.pre_fault_allowed) + return -EOPNOTSUPP; + + return 0; +} + long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { @@ -5105,23 +5165,12 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, u64 end; int r; - if (!vcpu->kvm->arch.pre_fault_allowed) - return -EOPNOTSUPP; - if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa))) return -EINVAL; - /* - * reload is efficient when called repeatedly, so we can do it on - * every iteration. - */ - r = kvm_mmu_reload(vcpu); - if (r) - return r; - direct_bits = 0; if (kvm_arch_has_private_mem(vcpu->kvm) && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa))) error_code |= PFERR_PRIVATE_ACCESS; else direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm)); @@ -5223,14 +5272,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn)) return -EPERM; - r = kvm_mmu_reload(vcpu); - if (r) - return r; - - r = mmu_topup_memory_caches(vcpu, false); - if (r) - return r; - do { if (signal_pending(current)) return -EINTR; @@ -5238,10 +5279,38 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + + r = mmu_topup_memory_caches(vcpu, false); + if (r) + return r; + cond_resched(); guard(read_lock)(&kvm->mmu_lock); + /* + * Because slots_lock is held, it should be impossible for *any* + * roots to be invalidated after the initial MMU reload. WARN, + * but continue on; the above MMU reload will do the right thing + * if the current root is actually invalid. + */ + WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)); + + /* + * Snapshot the invalidation sequence counter after acquiring + * mmu_lock, as guest_memfd guarantees the validity of the pfn, + * i.e. any concurrent invalidations are guaranteed to be + * irrelevant. + */ + fault.mmu_seq = vcpu->kvm->mmu_invalidate_seq; + if (is_page_fault_stale(vcpu, &fault)) { + r = RET_PF_RETRY; + continue; + } + r = kvm_tdp_mmu_map(vcpu, &fault); } while (r == RET_PF_RETRY); @@ -5920,6 +5989,9 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu, return role; } + if (KVM_BUG_ON(____is_efer_lma(regs) && !____is_cr4_pae(regs), vcpu->kvm)) + *(u64 *)®s->efer &= ~EFER_LMA; + role.base.efer_nx = ____is_efer_nx(regs); role.base.cr0_wp = ____is_cr0_wp(regs); role.base.cr4_smep = ____is_cr4_smep(regs); @@ -5963,19 +6035,22 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { - int maxpa; - - if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM) - maxpa = cpuid_query_maxguestphyaddr(vcpu); - else - maxpa = cpuid_maxphyaddr(vcpu); - /* tdp_root_level is architecture forced level, use it if nonzero */ if (tdp_root_level) return tdp_root_level; + /* + * If the VM has mirror roots, then the root level is predefined as the + * mirror root (and by extension the normal root) needs to match the + * root level that was configured for the external page tables that are + * being mirrored by KVM. + */ + if (kvm_has_mirrored_tdp(vcpu->kvm) && + !WARN_ON_ONCE(!vcpu->kvm->arch.mirror_root_level)) + return vcpu->kvm->arch.mirror_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ - if (max_tdp_level == 5 && maxpa <= 48) + if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; return max_tdp_level; @@ -6594,13 +6669,13 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && !(error_code & PFERR_RSVD_MASK) && vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) error_code |= PFERR_PRIVATE_ACCESS; r = RET_PF_INVALID; if (unlikely(error_code & PFERR_RSVD_MASK)) { - if (WARN_ON_ONCE(error_code & PFERR_PRIVATE_ACCESS)) - return -EFAULT; + if (KVM_BUG_ON(error_code & PFERR_PRIVATE_ACCESS, vcpu->kvm)) + return -EIO; r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); if (r == RET_PF_EMULATE) @@ -6642,6 +6717,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err return r; emulate: + if (error_code & PFERR_WRITE_MASK) + emulation_type |= EMULTYPE_PF_WRITE; + return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, insn_len); } @@ -6714,7 +6792,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, if (is_noncanonical_invlpg_address(addr, vcpu)) return; - kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL); + kvm_x86_call(flush_tlb_gva)(vcpu, addr); if (tdp_enabled) return; @@ -6844,7 +6922,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct k * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit * KVM; that horror is handled on-demand by mmu_alloc_special_roots(). */ - if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && kvm_mmu_get_max_tdp_level() > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); @@ -7127,6 +7205,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (WARN_ON_ONCE(gfn_end <= gfn_start)) return; + guard(srcu)(&kvm->srcu); + write_lock(&kvm->mmu_lock); kvm_mmu_invalidate_start(kvm); @@ -7995,7 +8075,7 @@ static void kvm_recover_nx_huge_pages(struct kvm *kvm, WARN_ON_ONCE(!sp->nx_huge_page_disallowed); WARN_ON_ONCE(!sp->role.direct); - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, mmu_type); if (is_tdp_mmu) spin_unlock(tdp_mmu_pages_lock); @@ -8107,7 +8187,7 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread); } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level) { @@ -8126,8 +8206,8 @@ static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; } -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { struct kvm_memory_slot *slot = range->slot; int level; @@ -8196,18 +8276,18 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, const unsigned long end = start + KVM_PAGES_PER_HPAGE(level); if (level == PG_LEVEL_2M) - return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs); + return kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attrs); for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) { if (hugepage_test_mixed(slot, gfn, level - 1) || - attrs != kvm_get_memory_attributes(kvm, gfn)) + attrs != kvm_get_vm_memory_attributes(kvm, gfn)) return false; } return true; } -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { unsigned long attrs = range->arg.attributes; struct kvm_memory_slot *slot = range->slot; @@ -8300,7 +8380,7 @@ void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm, * be manually checked as the attributes may already be mixed. */ for (gfn = start; gfn < end; gfn += nr_pages) { - unsigned long attrs = kvm_get_memory_attributes(kvm, gfn); + unsigned long attrs = kvm_get_vm_memory_attributes(kvm, gfn); if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) hugepage_clear_mixed(slot, gfn, level);
diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 7e8195a..f12558d 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c
@@ -16,6 +16,8 @@ #include <linux/kvm_host.h> #include <linux/rculist.h> +#include <asm/kvm_page_track.h> + #include "mmu.h" #include "mmu_internal.h" #include "page_track.h" @@ -237,10 +239,11 @@ static int kvm_enable_external_write_tracking(struct kvm *kvm) * register the notifier so that event interception for the tracked guest * pages can be received. */ -int kvm_page_track_register_notifier(struct kvm *kvm, +int kvm_page_track_register_notifier(struct file *file, struct kvm_page_track_notifier_node *n) { struct kvm_page_track_notifier_head *head; + struct kvm *kvm = file_to_kvm_x86(file); int r; if (!kvm || kvm->mm != current->mm) @@ -252,7 +255,7 @@ int kvm_page_track_register_notifier(struct kvm *kvm, return r; } - kvm_get_kvm(kvm); + get_file(file); head = &kvm->arch.track_notifier_head; @@ -267,10 +270,11 @@ EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); * stop receiving the event interception. It is the opposed operation of * kvm_page_track_register_notifier(). */ -void kvm_page_track_unregister_notifier(struct kvm *kvm, +void kvm_page_track_unregister_notifier(struct file *file, struct kvm_page_track_notifier_node *n) { struct kvm_page_track_notifier_head *head; + struct kvm *kvm = file_to_kvm_x86(file); head = &kvm->arch.track_notifier_head; @@ -279,7 +283,7 @@ void kvm_page_track_unregister_notifier(struct kvm *kvm, write_unlock(&kvm->mmu_lock); synchronize_srcu(&head->track_srcu); - kvm_put_kvm(kvm); + fput(file); } EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); @@ -336,11 +340,12 @@ void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot) * add guest page to the tracking pool so that corresponding access on that * page will be intercepted. * - * @kvm: the guest instance we are interested in. + * @file: the VM file of the guest instance we are interested in. * @gfn: the guest page. */ -int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn) +int kvm_write_track_add_gfn(struct file *file, gfn_t gfn) { + struct kvm *kvm = file_to_kvm_x86(file); struct kvm_memory_slot *slot; int idx; @@ -366,11 +371,12 @@ EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn); * remove the guest page from the tracking pool which stops the interception * of corresponding access on that page. * - * @kvm: the guest instance we are interested in. + * @file: the VM file of the guest instance we are interested in. * @gfn: the guest page. */ -int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn) +int kvm_write_track_remove_gfn(struct file *file, gfn_t gfn) { + struct kvm *kvm = file_to_kvm_x86(file); struct kvm_memory_slot *slot; int idx;
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index e5450fa..dc86c2a 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -367,13 +367,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, pte_access = ~0; /* - * Queue a page fault for injection if this assertion fails, as callers - * assume that walker.fault contains sane info on a walk failure. I.e. - * avoid making the situation worse by inducing even worse badness - * between when the assertion fails and when KVM kicks the vCPU out to - * userspace (because the VM is bugged). + * Queue a page fault for injection if any of the below assertions fail, + * as callers assume that walker.fault contains sane info on a walk + * failure. I.e. avoid making the situation worse by inducing even + * worse badness between when the assertion fails and when KVM kicks + * the vCPU out to userspace (because the VM is bugged). */ - if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm)) + if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm) || + KVM_BUG_ON(walker->max_level > PT_MAX_FULL_LEVELS, vcpu->kvm)) goto error; ++walker->level; @@ -390,7 +391,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, offset = index * sizeof(pt_element_t); pte_gpa = gfn_to_gpa(table_gfn) + offset; - BUG_ON(walker->level < 1); + if (KVM_BUG_ON(walker->level < 1, vcpu->kvm)) + goto error; + walker->table_gfn[walker->level - 1] = table_gfn; walker->pte_gpa[walker->level - 1] = pte_gpa; @@ -774,8 +777,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access, base_gfn, fault->pfn, fault);
diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index dd3bb04..e91650af 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c
@@ -598,9 +598,19 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) return false; - return true; + /* + * EFER_LMSLE_MBZ is a "defeature" bit, i.e. is set when the CPU does + * *not* support long mode segment limits, and so is the only EFER + * check whose polarity is inverted: EFER.LMSLE is legal if and only if + * the guest does *not* have the defeature. + */ + if (efer & EFER_LMSLE && + guest_cpu_cap_has(vcpu, X86_FEATURE_EFER_LMSLE_MBZ)) + return false; + return true; } + bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { if (efer & ~kvm_caps.supported_efer_bits) @@ -1631,7 +1641,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (msr_info->host_initiated) { kvm_synchronize_tsc(vcpu, &data); } else if (!vcpu->arch.guest_tsc_protected) { - u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; + u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) - + vcpu->arch.l1_tsc_offset; adjust_tsc_offset_guest(vcpu, adj); vcpu->arch.ia32_tsc_adjust_msr += adj; }
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index d2fd47e..eef2f79 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c
@@ -662,7 +662,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; - int bit; + int bit, r; bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); @@ -676,12 +676,14 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) kvm_for_each_pmc(pmu, pmc, bit, bitmap) { /* - * If reprogramming fails, e.g. due to contention, re-set the - * reprogram bit, i.e. opportunistically try again on the next - * PMU refresh. Don't make a new request as doing so can stall - * the guest if reprogramming repeatedly fails. + * If reprogramming fails on a transient condition, e.g. due to + * contention, re-set the reprogram bit, i.e. opportunistically + * try again on the next PMU refresh. Don't make a new request + * as doing so can stall the guest if reprogramming repeatedly + * fails. */ - if (reprogram_counter(pmc)) + r = reprogram_counter(pmc); + if (r == -EBUSY || r == -ENOMEM) set_bit(pmc->idx, pmu->reprogram_pmi); }
diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 8f66438..3621f1a 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c
@@ -440,9 +440,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); +static bool kvm_is_valid_cr8(unsigned long cr8) +{ + return !(cr8 & CR8_RESERVED_BITS); +} + int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { - if (cr8 & CR8_RESERVED_BITS) + if (!kvm_is_valid_cr8(cr8)) return 1; if (lapic_in_kernel(vcpu)) kvm_lapic_set_tpr(vcpu, cr8); @@ -565,6 +570,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4) && kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_is_valid_cr8(sregs->cr8) && kvm_valid_efer(vcpu, sregs->efer); } @@ -766,17 +772,19 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - u64 fixed = DR6_FIXED_1; + unsigned long fixed = DR6_ACTIVE_LOW & ~DR6_VOLATILE; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) fixed |= DR6_BUS_LOCK; + return fixed; } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr6_fixed_1); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) { @@ -792,7 +800,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) case 6: if (!kvm_dr6_valid(val)) return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_get_dr6_fixed_1(vcpu); break; case 5: default: /* 7 */
diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 447f0ec..97c17b1 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h
@@ -38,16 +38,14 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); #define DR6_BT (1 << 15) #define DR6_RTM (1 << 16) /* - * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. - * We can regard all the bits in DR6_FIXED_1 as active_low bits; - * they will never be 0 for now, but when they are defined - * in the future it will require no code change. + * DR6_ACTIVE_LOW combines fixed-1 and active-low bits (bits that are currently + * fixed-1 are guaranteed to have active-low semantics if they are ever used to + * support a new feature). * - * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. + * Note, DR6_ACTIVE_LOW is also the INIT/RESET value for DR6. */ #define DR6_ACTIVE_LOW 0xffff0ff0 #define DR6_VOLATILE 0x0001e80f -#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) #define DR7_BP_EN_MASK 0x000000ff #define DR7_GE (1 << 9) @@ -62,6 +60,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr); +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 2ad2578..89345af 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h
@@ -66,7 +66,7 @@ #define X86_FEATURE_AVX10_512 KVM_X86_FEATURE(CPUID_24_0_EBX, 18) /* Intel-defined sub-features, CPUID level 0x00000024:1 (ECX) */ -#define X86_FEATURE_AVX10_VNNI_INT KVM_X86_FEATURE(CPUID_24_1_ECX, 2) +#define X86_FEATURE_AVX10_V1_AUX KVM_X86_FEATURE(CPUID_24_1_ECX, 2) /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8)
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index f9090b6..af0ef67 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c
@@ -527,12 +527,16 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, /* Always clear misc_ctl bits that the guest cannot use */ to->misc_ctl = from->misc_ctl; + to->misc_ctl2 = from->misc_ctl2; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) to->misc_ctl &= ~SVM_MISC_ENABLE_NP; if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV)) + to->misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; + to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; to->tsc_offset = from->tsc_offset; @@ -550,7 +554,6 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->event_inj_err = from->event_inj_err; to->next_rip = from->next_rip; to->nested_cr3 = from->nested_cr3; - to->misc_ctl2 = from->misc_ctl2; to->pause_filter_count = from->pause_filter_count; to->pause_filter_thresh = from->pause_filter_thresh; @@ -735,12 +738,6 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, return 0; } -static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) -{ - return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); -} - static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) { struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; @@ -789,6 +786,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); + /* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */ + if (!(svm->nested.save.efer & EFER_LME)) + svm->nested.save.efer &= ~EFER_LMA; + svm_set_efer(vcpu, svm->nested.save.efer); svm_set_cr0(vcpu, svm->nested.save.cr0); @@ -807,17 +808,17 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; - svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; + svm->vcpu.arch.dr6 = svm->nested.save.dr6 | kvm_get_dr6_fixed_1(vcpu); vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (nested_vmcb12_has_lbrv(vcpu)) { + if (control->misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) { /* * Reserved bits of DEBUGCTL are ignored. Be consistent with * svm_set_msr's definition of reserved bits. */ svm_copy_lbrs(&vmcb02->save, save); - vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; + vmcb02->save.dbgctl &= svm_get_supported_debugctl(vcpu); } else { svm_copy_lbrs(&vmcb02->save, &vmcb01->save); } @@ -902,6 +903,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); /* + * PML is never enabled in hardware for L2. Make sure that an + * unexpected PML write would trigger a PML_FULL VM-Exit. + */ + if (pml) + vmcb02->control.pml_index = -1; + + /* * Stash vmcb02's counter if the guest hasn't moved past the guilty * instruction; otherwise, reset the counter to '0'. * @@ -1199,7 +1207,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) } /* Copy state save area fields which are handled by VMRUN */ -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save) { to_save->es = from_save->es; @@ -1226,7 +1234,7 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save, if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { svm_copy_lbrs(to_save, from_save); - to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; + to_save->dbgctl &= svm_get_supported_debugctl(vcpu); } } @@ -1297,7 +1305,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) vmcb12->control.next_rip = vmcb02->control.next_rip; - if (nested_vmcb12_has_lbrv(vcpu)) + if (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) svm_copy_lbrs(&vmcb12->save, &vmcb02->save); vmcb12->control.event_inj = 0; @@ -1374,7 +1382,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_exit_on_intr(svm)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - if (!nested_vmcb12_has_lbrv(vcpu)) { + if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)) { svm_copy_lbrs(&vmcb01->save, &vmcb02->save); vmcb_mark_dirty(vmcb01, VMCB_LBR); } @@ -1494,6 +1502,7 @@ int svm_allocate_nested(struct vcpu_svm *svm) if (!svm->nested.msrpm) goto err_free_vmcb02; + svm->nested.vmcb02.cpu = -1; svm->nested.initialized = true; return 0; @@ -1820,6 +1829,13 @@ int nested_svm_exit_special(struct vcpu_svm *svm) if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) return NESTED_EXIT_HOST; break; + case SVM_EXIT_PML_FULL: + /* + * All PML full exits are handled by KVM. KVM emulates PML in + * software for L1, but never enables PML in hardware on behalf + * of L1. + */ + return NESTED_EXIT_HOST; default: break; } @@ -2028,6 +2044,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, if (!(save->cr0 & X86_CR0_PG) || !(save->cr0 & X86_CR0_PE) || (save->rflags & X86_EFLAGS_VM) || + ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || !nested_vmcb_check_save(vcpu, &save_cached, false)) goto out_free; @@ -2064,7 +2081,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, save); nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02);
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 558eb01..b90c5f1 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c
@@ -573,6 +573,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, INIT_LIST_HEAD(&sev->regions_list); INIT_LIST_HEAD(&sev->mirror_vms); sev->need_init = false; + kvm->arch.has_protected_page_tables = true; kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); @@ -2058,12 +2059,14 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->pages_locked = src->pages_locked; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; + dst_kvm->arch.has_protected_page_tables = true; src->asid = 0; src->active = false; src->handle = 0; src->pages_locked = 0; src->es_active = false; + src_kvm->arch.has_protected_page_tables = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); @@ -2158,10 +2161,10 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (fd_empty(f)) return -EBADF; - if (!file_is_kvm(fd_file(f))) + source_kvm = file_to_kvm_x86(fd_file(f)); + if (!source_kvm) return -EBADF; - source_kvm = fd_file(f)->private_data; ret = sev_lock_two_vms(kvm, source_kvm); if (ret) return ret; @@ -2416,7 +2419,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, */ if (ret && !snp_page_reclaim(kvm, pfn) && sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && - sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) { + sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) { void *src_vaddr = kmap_local_page(src_page); void *dst_vaddr = kmap_local_pfn(pfn); @@ -2449,8 +2452,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) return -EFAULT; - pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__, - params.gfn_start, params.len, params.type, params.flags); + pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__, + params.gfn_start, params.len, params.type, params.flags, params.uaddr); if (!params.len || !PAGE_ALIGNED(params.len) || params.flags || (params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL && @@ -2462,7 +2465,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) src = NULL; - else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr)) + else if ((!gmem_in_place_conversion && !params.uaddr) || + !PAGE_ALIGNED(params.uaddr)) return -EINVAL; else src = u64_to_user_ptr(params.uaddr); @@ -2509,7 +2513,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.gfn_start += count; params.len -= count * PAGE_SIZE; - if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) + if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) params.uaddr += count * PAGE_SIZE; if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) @@ -2879,10 +2883,10 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (fd_empty(f)) return -EBADF; - if (!file_is_kvm(fd_file(f))) + source_kvm = file_to_kvm_x86(fd_file(f)); + if (!source_kvm) return -EBADF; - source_kvm = fd_file(f)->private_data; ret = sev_lock_two_vms(kvm, source_kvm); if (ret) return ret; @@ -2919,6 +2923,7 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) mutex_unlock(&sev_mirror_lock); mirror_sev->active = true; + kvm->arch.has_protected_page_tables = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; mirror_sev->es_active = source_sev->es_active; @@ -2977,6 +2982,7 @@ void sev_vm_init(struct kvm *kvm) kvm->arch.has_protected_state = true; fallthrough; case KVM_X86_SEV_VM: + kvm->arch.has_protected_page_tables = true; kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; to_kvm_sev_info(kvm)->need_init = true; break; @@ -4049,7 +4055,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); unsigned long mmu_seq; - struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); @@ -4093,7 +4098,7 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. */ - if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) + if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL)) return; read_lock(&kvm->mmu_lock); @@ -4109,8 +4114,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) else svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); read_unlock(&kvm->mmu_lock); - - kvm_release_page_clean(page); } /* @@ -4238,6 +4241,7 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ struct kvm *kvm = svm->vcpu.kvm; struct kvm_sev_info *sev = to_kvm_sev_info(kvm); sev_ret_code fw_err = 0; + u8 tickle = 0; int ret; if (!is_sev_snp_guest(&svm->vcpu)) @@ -4245,8 +4249,21 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ guard(mutex)(&sev->guest_req_mutex); - if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) - return -EIO; + if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } + + /* + * "Tickle" the response buffer to verify it's writable before sending + * the request to firmware, which will modify the VMPCK sequence number. + * Note, this is a best effort check and doesn't guard against TOCTOU + * issues. See below for more information. + */ + if (kvm_write_guest(kvm, resp_gpa, &tickle, sizeof(tickle))) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } data.gctx_paddr = __psp_pa(sev->snp_context); data.req_paddr = __psp_pa(sev->guest_req_buf); @@ -4261,6 +4278,13 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ if (ret && !fw_err) return ret; + /* + * Exit to userspace if writing the response fails, e.g. if the mapping + * changed between the initial tickle and the actual write, to avoid + * creating an ambiguous failure ABI with the guest. Don't return an + * error to the guest because firmware already incremented the VMPCK + * sequence number, but there's no way to communicate that to the guest. + */ if (kvm_write_guest(kvm, resp_gpa, sev->guest_resp_buf, PAGE_SIZE)) return -EIO; @@ -4312,8 +4336,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r return -EINVAL; if (kvm_read_guest(kvm, req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type), - &msg_type, 1)) - return -EIO; + &msg_type, 1)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } /* * As per GHCB spec, requests of type MSG_REPORT_REQ also allow for @@ -4352,8 +4378,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r * As per GHCB spec (see "SNP Extended Guest Request"), the * certificate table is terminated by 24-bytes of zeroes. */ - if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) - return -EIO; + if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } } return snp_handle_guest_req(svm, req_gpa, resp_gpa); @@ -5011,7 +5039,7 @@ struct page *snp_safe_alloc_page_node(int node, gfp_t gfp) * Allocate an SNP-safe page to workaround the SNP erratum where * the CPU will incorrectly signal an RMP violation #PF if a * hugepage (2MB or 1GB) collides with the RMP entry of a - * 2MB-aligned VMCB, VMSA, or AVIC backing page. + * 2MB-aligned VMCB, VMSA, PML or AVIC backing page. * * Allocate one extra page, choose a page which is not * 2MB-aligned, and free the other. @@ -5036,7 +5064,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) struct kvm_memory_slot *slot; struct kvm *kvm = vcpu->kvm; int order, rmp_level, ret; - struct page *page; + unsigned long mmu_seq; bool assigned; kvm_pfn_t pfn; gfn_t gfn; @@ -5050,7 +5078,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults * for shared pages should not end up here. */ - if (!kvm_mem_is_private(kvm, gfn)) { + if (!kvm_is_private_gfn(kvm, gfn)) { pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n", gpa); return; @@ -5063,7 +5091,10 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) return; } - ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order); + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + + ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order); if (ret) { pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n", gpa); @@ -5072,9 +5103,13 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); if (ret || !assigned) { - pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", - gpa, pfn, ret); - goto out_no_trace; + guard(read_lock)(&kvm->mmu_lock); + + if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", + gpa, pfn, ret); + + return; } /* @@ -5102,26 +5137,31 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) if (rmp_level == PG_LEVEL_4K) goto out; - ret = snp_rmptable_psmash(pfn); - if (ret) { - /* - * Look it up again. If it's 4K now then the PSMASH may have - * raced with another process and the issue has already resolved - * itself. - */ - if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && - assigned && rmp_level == PG_LEVEL_4K) + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) goto out; - pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", - gpa, pfn, ret); + ret = snp_rmptable_psmash(pfn); + if (ret) { + /* + * Look it up again. If it's 4K now then the PSMASH may + * have raced with another process and the issue has + * already resolved itself. If it's not assigned, then + * this must have raced with another process that made + * this page shared. + */ + if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && + ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) + goto out; + + pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", + gpa, pfn, ret); + } } kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD); out: trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret); -out_no_trace: - kvm_release_page_unused(page); } static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end)
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c054803..136069d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c
@@ -179,6 +179,9 @@ module_param(vnmi, bool, 0444); module_param(enable_mediated_pmu, bool, 0444); +bool __ro_after_init pml = true; +module_param(pml, bool, 0444); + static bool __ro_after_init svm_gp_erratum_intercept = true; static u8 rsm_ins_bytes[] = "\x0f\xaa"; @@ -530,6 +533,15 @@ static int svm_check_processor_compat(void) static void __svm_write_tsc_multiplier(u64 multiplier) { + /* + * Fallback to the default ratio if KVM is buggy and tries to program + * an unsupported scaling ratio, e.g. so that the guest has a chance of + * surviving, so that the cache isn't stale/corrupted, and so that KVM + * doesn't leak state across VMs. + */ + if (WARN_ON_ONCE(multiplier & SVM_TSC_RATIO_RSVD)) + multiplier = SVM_TSC_RATIO_DEFAULT; + if (multiplier == __this_cpu_read(current_tsc_ratio)) return; @@ -872,13 +884,22 @@ static void __svm_disable_lbrv(struct kvm_vcpu *vcpu) to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; } +static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (svm->vmcb->save.dbgctl & (DEBUGCTLMSR_LBR | DEBUGCTLMSR_BUS_LOCK_DETECT)) + return true; + + return is_guest_mode(vcpu) && + (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); +} + void svm_update_lbrv(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; - bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) || - (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)); + bool enable_lbrv = svm_need_lbr_virtualization(vcpu); if (enable_lbrv && !current_enable_lbrv) __svm_enable_lbrv(vcpu); @@ -1258,6 +1279,23 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) if (vcpu->kvm->arch.bus_lock_detection_enabled) svm_set_intercept(svm, INTERCEPT_BUSLOCK); + if (pml) { + /* + * Populate the page address and index here, PML is enabled + * when dirty logging is enabled on the memslot through + * svm_update_cpu_dirty_logging() + */ + control->pml_addr = (u64)__sme_set(page_to_phys(vcpu->arch.pml_page)); + + /* + * The VMCB control area remains valid after an intercepted SHUTDOWN + * (AMD APM Rev 3.45+), so only initialize PML index on reset to avoid + * discarding already-logged entries that haven't been flushed. + */ + if (!init_event) + control->pml_index = PML_HEAD_INDEX; + } + if (is_sev_guest(vcpu)) sev_init_vmcb(svm, init_event); @@ -1317,9 +1355,15 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) if (!vmcb01_page) goto out; + if (pml) { + vcpu->arch.pml_page = snp_safe_alloc_page(); + if (!vcpu->arch.pml_page) + goto error_free_vmcb_page; + } + err = sev_vcpu_create(vcpu); if (err) - goto error_free_vmcb_page; + goto error_free_pml_page; err = avic_init_vcpu(svm); if (err) @@ -1346,6 +1390,9 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) avic_vcpu_free(vcpu); error_free_sev: sev_free_vcpu(vcpu); +error_free_pml_page: + if (vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); error_free_vmcb_page: __free_page(vmcb01_page); out: @@ -1365,6 +1412,9 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) sev_free_vcpu(vcpu); + if (pml && vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); + __free_page(__sme_pa_to_page(svm->vmcb01.pa)); svm_vcpu_free_msrpm(svm->msrpm); } @@ -2753,10 +2803,19 @@ static int efer_trap(struct kvm_vcpu *vcpu) * bit in svm_set_efer(), but __kvm_valid_efer() checks it against * whether the guest has X86_FEATURE_SVM - this avoids a failure if * the guest doesn't have X86_FEATURE_SVM. + * + * Clear EFER_LMSLE for a related reason: EFER writes are *trapped*, + * not intercepted, i.e. hardware has already committed the write by + * the time KVM gains control, and the trap is enabled if and only if + * the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so + * can't be fixed up by KVM. Rejecting EFER.LMSLE=1 would inject a #GP + * *and* leave EFER.LMSLE set in the guest, which is strictly worse + * than honoring a write that hardware itself allowed. */ msr_info.host_initiated = false; msr_info.index = MSR_EFER; - msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; + msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & + ~(EFER_SVME | EFER_LMSLE); ret = kvm_set_msr_common(vcpu, &msr_info); return kvm_complete_insn_gp(vcpu, ret); @@ -3169,7 +3228,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) data &= ~DEBUGCTLMSR_BTF; } - if (data & DEBUGCTL_RESERVED_BITS) + if (data & ~svm_get_supported_debugctl(vcpu)) return 1; if (svm->vmcb->save.dbgctl == data) @@ -3338,6 +3397,45 @@ static int vmmcall_interception(struct kvm_vcpu *vcpu) return kvm_emulate_hypercall(vcpu); } +static void svm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb01 = svm->vmcb01.ptr; + + if (enable) + vmcb01->control.misc_ctl |= SVM_MISC_ENABLE_PML; + else + vmcb01->control.misc_ctl &= ~SVM_MISC_ENABLE_PML; + + vmcb_mark_dirty(vmcb01, VMCB_NPT); +} + +static void svm_flush_pml_buffer(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_control_area *control = &svm->vmcb->control; + + /* Do nothing if PML buffer is empty */ + if (control->pml_index == PML_HEAD_INDEX) + return; + + kvm_flush_pml_buffer(vcpu, control->pml_index); + + /* Reset the PML index */ + control->pml_index = PML_HEAD_INDEX; +} + +static int pml_full_interception(struct kvm_vcpu *vcpu) +{ + trace_kvm_pml_full(vcpu->vcpu_id); + + /* + * PML buffer is already flushed at the beginning of svm_handle_exit(). + * Nothing to do here. + */ + return 1; +} + static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -3415,6 +3513,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { #ifdef CONFIG_KVM_AMD_SEV [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, #endif + [SVM_EXIT_PML_FULL] = pml_full_interception, }; static void dump_vmcb(struct kvm_vcpu *vcpu) @@ -3464,8 +3563,14 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); - pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl); + pr_err("%-20s%llx\n", "misc_ctl:", control->misc_ctl); pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); + + if (pml) { + pr_err("%-20s%016llx\n", "pml_addr:", control->pml_addr); + pr_err("%-20s%04x\n", "pml_index:", control->pml_index); + } + pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); pr_err("%-20s%08x\n", "event_inj:", control->event_inj); @@ -3643,6 +3748,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code) (u64)exit_code != __exit_code) goto unexpected_vmexit; + /* + * PML is never enabled when running L2, bail immediately if a PML full + * exit occurs as something is horribly wrong. + */ + if (unlikely(is_guest_mode(vcpu) && exit_code == SVM_EXIT_PML_FULL)) + goto unexpected_vmexit; + #ifdef CONFIG_MITIGATION_RETPOLINE if (exit_code == SVM_EXIT_MSR) return msr_interception(vcpu); @@ -3711,6 +3823,14 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; + /* + * Opportunistically flush the PML buffer on VM exit. This keeps the + * dirty bitmap current by processing logged GPAs rather than waiting for + * PML_FULL exit. + */ + if (vcpu->kvm->arch.cpu_dirty_log_size && !is_guest_mode(vcpu)) + svm_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) return 0; @@ -3759,14 +3879,25 @@ static int pre_svm_run(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); /* - * If the previous vmrun of the vmcb occurred on a different physical - * cpu, then mark the vmcb dirty and assign a new asid. Hardware's - * vmcb clean bits are per logical CPU, as are KVM's asid assignments. + * If the previous VMRUN of the VMCB occurred on a different physical + * cpu, then mark the VMCB dirty as hardware's clean bits are per pCPU. + * + * Reset the ASID generation in both VMCBs. This will lead to assigning + * a new ASID now, and then again when switching to the other VMCB. + * However, this is needed as the ASID is shared between the VMCBs, and + * otherwise it would be possible to use an ASID allocated on one pCPU + * on another, for example: + * - vCPU migrates from pCPU A to pCPU B, allocates a new ASID. + * - vCPU migrates back to pCPU A, and then switches the VMCB. + * - The new VMCB does not detect a pCPU change and runs on pCPU A with + * the new ASID allocated on pCPU B, which is potentially used by + * another vCPU/VM. */ if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { - svm->current_vmcb->asid_generation = 0; vmcb_mark_all_dirty(svm->vmcb); svm->current_vmcb->cpu = vcpu->cpu; + svm->vmcb01.asid_generation = 0; + svm->nested.vmcb02.asid_generation = 0; } if (is_sev_guest(vcpu)) @@ -4239,7 +4370,7 @@ static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) +static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) { struct vcpu_svm *svm = to_svm(vcpu); @@ -4255,8 +4386,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) } svm_flush_tlb_guest(vcpu); - if (full) - *full = true; } static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) @@ -5053,7 +5182,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); - svm_copy_vmrun_state(m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); + svm_copy_vmrun_state(vcpu, m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); return 0; } @@ -5093,7 +5222,7 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) * used during SMM (see svm_enter_smm()) */ - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, m_save.map.hva + 0x400); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400); /* * Enter the nested guest now @@ -5312,6 +5441,9 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); + if (pml) + kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; + svm_srso_vm_init(); return 0; } @@ -5430,6 +5562,8 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, + .update_cpu_dirty_logging = svm_update_cpu_dirty_logging, + .deliver_interrupt = svm_deliver_interrupt, .pi_update_irte = avic_pi_update_irte, .setup_mce = svm_setup_mce, @@ -5572,6 +5706,16 @@ static __init void svm_set_cpu_caps(void) boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + /* + * Tell userspace that EFER.LMSLE must be zero if nested SVM is + * disabled, as KVM allows EFER.LMSLE if and only if nested SVM is + * supported (a historical artifact of commit eec4b140c924 ("KVM: SVM: + * Allow EFER.LMSLE to be set with nested svm"), not an architectural + * requirement). + */ + if (!nested) + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + if (enable_pmu) { /* * Enumerate support for PERFCTR_CORE if and only if KVM has @@ -5596,9 +5740,17 @@ static __init void svm_set_cpu_caps(void) * Clear capabilities that are automatically configured by common code, * but that require explicit SVM support (that isn't yet implemented). */ - kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + /* + * LBR Virtualization must be enabled to support BusLockTrap inside the + * guest, since BusLockTrap is enabled through MSR_IA32_DEBUGCTLMSR and + * MSR_IA32_DEBUGCTLMSR is virtualized only if LBR Virtualization is + * enabled. + */ + if (!lbrv) + kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } @@ -5621,7 +5773,8 @@ static __init int svm_hardware_setup(void) XFEATURE_MASK_BNDCSR); if (tsc_scaling) { - if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { + if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR) || + !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { tsc_scaling = false; } else { pr_info("TSC scaling supported\n"); @@ -5690,6 +5843,10 @@ static __init int svm_hardware_setup(void) nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); + pml = pml && npt_enabled && cpu_feature_enabled(X86_FEATURE_PML); + if (pml) + pr_info("Page modification logging supported\n"); + if (lbrv) { if (!boot_cpu_has(X86_FEATURE_LBRV)) lbrv = false;
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 29bde74..6d02429 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h
@@ -53,6 +53,7 @@ extern int vgif; extern bool intercept_smi; extern bool vnmi; extern int lbrv; +extern bool pml; extern int tsc_aux_uret_slot __ro_after_init; @@ -786,7 +787,15 @@ BUILD_SVM_MSR_BITMAP_HELPERS(bool, test, test) BUILD_SVM_MSR_BITMAP_HELPERS(void, clear, __clear) BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) -#define DEBUGCTL_RESERVED_BITS (~DEBUGCTLMSR_LBR) +static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) +{ + u64 debugctl = DEBUGCTLMSR_LBR; + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; + + return debugctl; +} /* svm.c */ extern bool dump_invalid_vmcb; @@ -876,7 +885,7 @@ void svm_leave_nested(struct kvm_vcpu *vcpu); void svm_free_nested(struct vcpu_svm *svm); int svm_allocate_nested(struct vcpu_svm *svm); int nested_svm_vmrun(struct kvm_vcpu *vcpu); -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save); void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb); void nested_svm_vmexit(struct vcpu_svm *svm);
diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 0800567..b290533 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h
@@ -2,10 +2,13 @@ #ifndef __KVM_X86_VMX_COMMON_H #define __KVM_X86_VMX_COMMON_H +#include <linux/container_of.h> #include <linux/kvm_host.h> #include <asm/posted_intr.h> +#include <asm/vmx.h> #include "mmu.h" +#include "vmx_ops.h" union vmx_exit_reason { struct { @@ -55,6 +58,8 @@ struct vcpu_vt { #endif }; +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu); + #ifdef CONFIG_KVM_INTEL_TDX static __always_inline bool is_td(struct kvm *kvm) @@ -74,14 +79,62 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + +static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) +{ + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); +} + +static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) +{ + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); +} + +static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu) +{ + return to_vt(vcpu)->exit_reason; +} + +static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + + return vt->exit_qualification; +} + +static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + return vt->exit_intr_info; +} + +static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->xfd && + !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); +} + static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) { /* For TDX the direct mask is the shared mask. */ return !kvm_is_addr_direct(kvm, gpa); } -static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, - unsigned long exit_qualification) +static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, + unsigned long exit_qualification) { u64 error_code; @@ -164,8 +217,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the * interrupt if necessary. */ -static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, - struct pi_desc *pi_desc, int vector) +static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu, + struct pi_desc *pi_desc, int vector) { if (pi_test_and_set_pir(vector, pi_desc)) return; @@ -183,6 +236,33 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); +static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, + unsigned long exit_qual) +{ + bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; + + ++vcpu->stat.notify_window_exits; + + if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || + context_invalid) { + vcpu->run->exit_reason = KVM_EXIT_NOTIFY; + vcpu->run->notify.flags = context_invalid ? + KVM_NOTIFY_CONTEXT_INVALID : 0; + return 0; + } + + return 1; +} + +static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} #endif /* __KVM_X86_VMX_COMMON_H */
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8..d75170a 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c
@@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/entry-common.h> #include <linux/moduleparam.h> #include "x86_ops.h" @@ -10,8 +11,12 @@ #include "tdx.h" #include "tdx_arch.h" +#pragma GCC poison to_vmx + +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt)); + #ifdef CONFIG_KVM_INTEL_TDX -static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt)); static void vt_disable_virtualization_cpu(void) { @@ -108,7 +113,7 @@ static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load(vcpu, cpu); } -static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { /* * Basic TDX does not support feature PML. KVM does not enable PML in @@ -117,7 +122,7 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(is_td_vcpu(vcpu))) return; - vmx_update_cpu_dirty_logging(vcpu); + vmx_update_cpu_dirty_logging(vcpu, enable); } static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) @@ -154,15 +159,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) return vmx_vcpu_run(vcpu, run_flags); } -static int vt_handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion fastpath) -{ - if (is_td_vcpu(vcpu)) - return tdx_handle_exit(vcpu, fastpath); - - return vmx_handle_exit(vcpu, fastpath); -} - static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu) { if (is_td_vcpu(vcpu)) { @@ -543,12 +539,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) vmx_flush_tlb_current(vcpu); } -static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { if (is_td_vcpu(vcpu)) return; - vmx_flush_tlb_gva(vcpu, addr, full); + vmx_flush_tlb_gva(vcpu, addr); } static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) @@ -601,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) vmx_set_nmi_mask(vcpu, masked); } +static void vt_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* + * TDX doesn't support disabling HLT-exiting, and KVM can't access a + * TD's VMCS, so there is never any hardware halted state to clear. + */ + if (is_td_vcpu(vcpu)) + return; + + vmx_clear_hlt(vcpu); +} + static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) { /* Refer to the comments in tdx_inject_nmi(). */ @@ -807,7 +815,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) vmx_write_tsc_multiplier(vcpu); } -#ifdef CONFIG_X86_64 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired) { @@ -826,7 +833,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) vmx_cancel_hv_timer(vcpu); } -#endif static void vt_setup_mce(struct kvm_vcpu *vcpu) { @@ -876,6 +882,112 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) +{ + if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vt_get_intr_info(vcpu))) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); + x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); + kvm_after_interrupt(vcpu); +} + +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM + * interception may have been caused by L1 interception. Per the SDM, + * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. + * + * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. + * unlike CR2 and DR6, the value is not a payload that is attached to + * the #NM exception. + */ + if (is_xfd_nm_fault(vcpu)) + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + +static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +{ + /* if exit due to PF check for async PF */ + if (is_page_fault(intr_info)) + vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(vcpu); + /* Handle machine checks before interrupts are enabled */ + else if (is_machine_check(intr_info)) + kvm_machine_check(); +} + +static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, + u32 intr_info) +{ + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + + if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, + "unexpected VM-Exit interrupt info: 0x%x", intr_info)) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); + x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); + kvm_after_interrupt(vcpu); + + vcpu->arch.at_instruction_boundary = true; +} + +static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) +{ + if (to_vt(vcpu)->emulation_required) + return; + + switch (vt_get_exit_reason(vcpu).basic) { + case EXIT_REASON_EXTERNAL_INTERRUPT: + handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu)); + break; + case EXIT_REASON_EXCEPTION_NMI: + handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu)); + break; + case EXIT_REASON_MCE_DURING_VMENTRY: + kvm_machine_check(); + break; + default: + break; + } +} + +static int vt_handle_exit(struct kvm_vcpu *vcpu, + enum exit_fastpath_completion fastpath) +{ + int ret; + +#ifdef CONFIG_KVM_INTEL_TDX + if (is_td_vcpu(vcpu)) + ret = tdx_handle_exit(vcpu, fastpath); + else +#endif + ret = vmx_handle_exit(vcpu, fastpath); + + /* + * Exit to user space when bus lock detected to inform that there is + * a bus lock in guest. + */ + if (vt_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ @@ -949,7 +1061,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization), .vcpu_run = vt_op(vcpu_run), - .handle_exit = vt_op(handle_exit), + .handle_exit = vt_handle_exit, .skip_emulated_instruction = vmx_skip_emulated_instruction, .update_emulated_instruction = vmx_update_emulated_instruction, .unhandleable_emulation_required = vt_op(unhandleable_emulation_required), @@ -964,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .nmi_allowed = vt_op(nmi_allowed), .get_nmi_mask = vt_op(get_nmi_mask), .set_nmi_mask = vt_op(set_nmi_mask), + .clear_hlt = vt_op(clear_hlt), .enable_nmi_window = vt_op(enable_nmi_window), .enable_irq_window = vt_op(enable_irq_window), .update_cr8_intercept = vt_op(update_cr8_intercept), @@ -1000,12 +1113,12 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .load_mmu_pgd = vt_op(load_mmu_pgd), .check_intercept = vmx_check_intercept, - .handle_exit_irqoff = vmx_handle_exit_irqoff, + .handle_exit_irqoff = vt_handle_exit_irqoff, .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), - .pi_update_irte = vmx_pi_update_irte, - .pi_start_bypass = vmx_pi_start_bypass, + .pi_update_irte = vt_pi_update_irte, + .pi_start_bypass = vt_pi_start_bypass, #ifdef CONFIG_X86_64 .set_hv_timer = vt_op(set_hv_timer),
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 5165275..516ba03 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c
@@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * "NMI unblocking due to IRET", i.e. the bit can be propagated * as-is from the original EXIT_QUALIFICATION. */ - exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; + exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; } else { if (fault->error_code & PFERR_RSVD_MASK) { vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; @@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * and walks the faulting GPA. */ if (from_hardware) - exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + exit_qualification |= vt_get_exit_qual(vcpu) & mask; else exit_qualification |= fault->exit_qualification & mask; @@ -4750,7 +4750,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, { /* update exit information fields: */ vmcs12->vm_exit_reason = vm_exit_reason; - if (vmx_get_exit_reason(vcpu).enclave_mode) + if (vt_get_exit_reason(vcpu).enclave_mode) vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; vmcs12->exit_qualification = exit_qualification; @@ -5376,7 +5376,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, struct x86_exception e; int r; - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, sizeof(*vmpointer), &gva)) { *ret = 1; @@ -5668,7 +5668,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5774,7 +5774,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5900,6 +5900,12 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; + /* Forbid normal VMPTRLD if Enlightened version was used */ + if (nested_vmx_is_evmptr12_valid(vmx)) { + kvm_queue_exception(vcpu, UD_VECTOR); + return 1; + } + if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) return r; @@ -5909,10 +5915,6 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) if (vmptr == vmx->nested.vmxon_ptr) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); - /* Forbid normal VMPTRLD if Enlightened version was used */ - if (nested_vmx_is_evmptr12_valid(vmx)) - return 1; - if (vmx->nested.current_vmptr != vmptr) { struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; struct vmcs_hdr hdr; @@ -5963,9 +5965,9 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); - gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; + gpa_t current_vmptr; struct x86_exception e; gva_t gva; int r; @@ -5973,8 +5975,16 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; - if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))) - return 1; + /* + * Hyper-V TLFS does not specify the behavior of VMPTRST when eVMCS is used + * but genuine Hyper-V seems to be returning eVMCS GPA. + */ +#ifdef CONFIG_KVM_HYPERV + if (nested_vmx_is_evmptr12_valid(to_vmx(vcpu))) + current_vmptr = to_vmx(vcpu)->nested.hv_evmcs_vmptr; + else +#endif + current_vmptr = to_vmx(vcpu)->nested.current_vmptr; if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, sizeof(gpa_t), &gva)) @@ -6024,7 +6034,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) /* According to the Intel VMX instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6107,7 +6117,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* according to the intel vmx instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6239,8 +6249,8 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) * EXIT_REASON_VMFUNC as the exit reason. */ nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, - vmx_get_intr_info(vcpu), - vmx_get_exit_qual(vcpu)); + vt_get_intr_info(vcpu), + vt_get_exit_qual(vcpu)); return 1; } @@ -6291,7 +6301,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -6317,7 +6327,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) - msr_index = vmx_get_exit_qual(vcpu); + msr_index = vt_get_exit_qual(vcpu); else msr_index = kvm_ecx_read(vcpu); @@ -6353,7 +6363,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int cr = exit_qualification & 15; int reg; unsigned long val; @@ -6487,7 +6497,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6570,7 +6580,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6740,14 +6750,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) * need to be synthesized by querying the in-kernel LAPIC, but external * interrupts are never reflected to L1 so it's a non-issue. */ - exit_intr_info = vmx_get_intr_info(vcpu); + exit_intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(exit_intr_info)) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); reflect_vmexit: nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);
diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 4a6d9a1..2644add 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c
@@ -14,6 +14,8 @@ #include "vmx.h" #include "tdx.h" +#pragma GCC poison to_vmx + /* * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when @@ -54,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new) return 0; } -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); struct vcpu_vt *vt = to_vt(vcpu); @@ -144,11 +146,11 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) pi_set_on(pi_desc); } -static bool vmx_can_use_vtd_pi(struct kvm *kvm) +static bool vt_can_use_vtd_pi(struct kvm *kvm) { /* * Note, reading the number of possible bypass IRQs can race with a - * bypass IRQ being attached to the VM. vmx_pi_start_bypass() ensures + * bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK. */ return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() && @@ -206,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); } -static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) +static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu) { /* * The default posted interrupt vector does nothing when @@ -217,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) * back to the pi_wakeup_handler() function. */ return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) || - vmx_can_use_vtd_pi(vcpu->kvm); + vt_can_use_vtd_pi(vcpu->kvm); } -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - if (!vmx_needs_pi_wakeup(vcpu)) + if (!vt_needs_pi_wakeup(vcpu)) return; /* @@ -292,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) /* * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup - * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put(). + * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put(). */ -void vmx_pi_start_bypass(struct kvm *kvm) +void vt_pi_start_bypass(struct kvm *kvm) { - if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm))) + if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm))) return; kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); } -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector) {
diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h index a4af399..c5860ea 100644 --- a/arch/x86/kvm/vmx/posted_intr.h +++ b/arch/x86/kvm/vmx/posted_intr.h
@@ -8,16 +8,16 @@ #include <asm/posted_intr.h> -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu); +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu); void pi_wakeup_handler(void); void __init pi_init_cpu(int cpu); void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu); bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu); -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector); -void vmx_pi_start_bypass(struct kvm *kvm); +void vt_pi_start_bypass(struct kvm *kvm); static inline int pi_find_highest_vector(struct pi_desc *pi_desc) {
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index aac79fe..8a59315 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c
@@ -56,9 +56,6 @@ bool enable_tdx __ro_after_init; module_param_named(tdx, enable_tdx, bool, 0444); -#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) -#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) - static const struct tdx_sys_info *tdx_sysinfo; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err) @@ -619,6 +616,7 @@ int tdx_vm_init(struct kvm *kvm) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm->arch.has_protected_page_tables = true; kvm->arch.has_protected_state = true; /* * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap, @@ -718,7 +716,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm))) return; @@ -743,7 +741,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu) * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT, * which passes the interrupt blocked flag. */ - return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || !to_tdx(vcpu)->vp_enter_args.r12; } @@ -761,7 +759,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) * otherwise the interrupt would have been serviced at the instruction * boundary. */ - if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || to_tdx(vcpu)->vp_enter_args.r12) return false; @@ -828,7 +826,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) void tdx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); tdx_prepare_switch_to_host(vcpu); } @@ -925,36 +923,51 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu) return EXIT_REASON_TDCALL; } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) { - struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; - - switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { + switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { case TDX_SUCCESS: case TDX_NON_RECOVERABLE_VCPU: case TDX_NON_RECOVERABLE_TD: case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE: case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE: - break; + return true; default: - return -1u; + return false; } +} - exit_reason = tdx->vp_enter_ret; +static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +{ + struct vcpu_tdx *tdx = to_tdx(vcpu); + union vmx_exit_reason exit_reason; - switch (exit_reason) { + /* + * Return the synthesized invalid Exit Reason, as the TDX module + * never attempted to run the vCPU, i.e. the Exit Reason is undefined, + * but this is NOT a failed VM-Enter. + */ + if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) + return (union vmx_exit_reason) { + .basic = EXIT_REASON_UNDEFINED, + }; + + exit_reason.full = (u32)tdx->vp_enter_ret; + + switch (exit_reason.basic) { case EXIT_REASON_TDCALL: if (tdvmcall_exit_type(vcpu)) - return EXIT_REASON_VMCALL; - - return tdcall_to_vmx_exit_reason(vcpu); + exit_reason.basic = EXIT_REASON_VMCALL; + else + exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu); + break; case EXIT_REASON_EPT_MISCONFIG: /* * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return -1u; + exit_reason.basic = EXIT_REASON_UNDEFINED; + break; default: break; } @@ -971,22 +984,21 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args); - vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu); + vt->exit_reason = tdx_to_vmx_exit_reason(vcpu); vt->exit_qualification = tdx->vp_enter_args.rcx; tdx->ext_exit_qualification = tdx->vp_enter_args.rdx; tdx->exit_gpa = tdx->vp_enter_args.r8; vt->exit_intr_info = tdx->vp_enter_args.r9; - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); guest_state_exit_irqoff(); } static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry && - vmx_get_exit_reason(vcpu).full != -1u; + return vt_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) @@ -1075,8 +1087,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * allowing vCPU entry to avoid contention with tdh_vp_enter() and * TDCALLs. */ - if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) + if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) { + /* + * The vCPU never entered the guest, but this looks like a + * handled exit to the caller. Synthesize an invalid exit + * reason so the previous exit's stale value isn't consumed + * a second time. + */ + vt->exit_reason.full = EXIT_REASON_UNDEFINED; return EXIT_FASTPATH_EXIT_HANDLED; + } trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT); @@ -1135,7 +1155,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu) static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu) { - u32 intr_info = vmx_get_intr_info(vcpu); + u32 intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -1613,10 +1633,7 @@ static int handle_tdvmcall(struct kvm_vcpu *vcpu) void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) { - u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 : - TDX_SHARED_BIT_PWL_4; - - if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm)) + if (KVM_BUG_ON(pgd_level != vcpu->kvm->arch.mirror_root_level, vcpu->kvm)) return; td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); @@ -1744,7 +1761,7 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve return ret; /* - * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() + * Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed() * before kvm_tdx->state. Userspace must not be allowed to pre-fault * arbitrary memory until the initial memory image is finalized. Pairs * with the smp_wmb() in tdx_td_finalize(). @@ -1939,7 +1956,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, struct vcpu_tdx *tdx = to_tdx(vcpu); /* TDX supports only posted interrupt. No lapic emulation. */ - __vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); } @@ -1947,7 +1964,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu) { u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK; - u64 eq = vmx_get_exit_qual(vcpu); + u64 eq = vt_get_exit_qual(vcpu); if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION) return false; @@ -1983,7 +2000,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) /* Only private GPA triggers zero-step mitigation */ local_retry = true; } else { - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); /* * EPT violation due to instruction fetch should never be * triggered from shared memory in TDX guest. If such EPT @@ -2031,7 +2048,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) while (1) { struct kvm_memory_slot *slot; - ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual); + ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual); if (ret != RET_PF_RETRY || !local_retry) break; @@ -2065,7 +2082,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) return 1; } - if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) + if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu)); return 1; @@ -2076,7 +2093,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); if (fastpath != EXIT_FASTPATH_NONE) return 1; @@ -2170,6 +2187,11 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) * - If it's not an MSMI, no need to do anything here. */ return 1; + case EXIT_REASON_NOTIFY: + /* NMI blocking state is handled by TDX module */ + return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu)); + case EXIT_REASON_BUS_LOCK: + return vt_handle_bus_lock_vmexit(vcpu); default: break; } @@ -2185,10 +2207,10 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_tdx *tdx = to_tdx(vcpu); *reason = tdx->vt.exit_reason.full; - if (*reason != -1u) { - *info1 = vmx_get_exit_qual(vcpu); + if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { + *info1 = vt_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); } else { *info1 = 0; *info2 = 0; @@ -2794,6 +2816,14 @@ DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t, if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T), tdx_acquire_vm_state_locks(kvm), struct kvm *kvm); +static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level) +{ + BUILD_BUG_ON(level != 4 && level != 5); + + kvm->arch.mirror_root_level = level; + kvm->arch.gfn_direct_bits = gpa_to_gfn(BIT_ULL(level == 4 ? 47 : 51)); +} + static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) { struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data); @@ -2857,9 +2887,9 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->xfam = td_params->xfam; if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; + tdx_set_mirror_root_level(kvm, 5); else - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + tdx_set_mirror_root_level(kvm, 4); kvm_tdx->state = TD_STATE_INITIALIZED; out: @@ -3186,6 +3216,17 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc)); td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR); + if (kvm_notify_vmexit_enabled(vcpu->kvm)) { + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_NOTIFY_VM_EXITING); + td_vmcs_write32(tdx, NOTIFY_WINDOW, + vcpu->kvm->arch.notify_window); + } + + if (vcpu->kvm->arch.bus_lock_detection_enabled) + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_BUS_LOCK_DETECTION); + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; @@ -3221,7 +3262,7 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - kvm_tdx->page_add_src = src_page; + kvm_tdx->page_add_src = src_page ?: pfn_to_page(pfn); ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; @@ -3251,6 +3292,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; + gpa_t nr_bytes, end_gpa; long gmem_ret; int ret; @@ -3267,11 +3309,15 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (copy_from_user(®ion, u64_to_user_ptr(cmd->data), sizeof(region))) return -EFAULT; - if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || - !PAGE_ALIGNED(region.gpa) || !region.nr_pages || - region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || + if (!PAGE_ALIGNED(region.source_addr) || + (!gmem_in_place_conversion && !region.source_addr) || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages) + return -EINVAL; + + if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || + check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) + !vt_is_tdx_private_gpa(kvm, end_gpa)) return -EINVAL; ret = 0; @@ -3298,7 +3344,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c break; } - region.source_addr += PAGE_SIZE; + if (region.source_addr) + region.source_addr += PAGE_SIZE; region.gpa += PAGE_SIZE; region.nr_pages--;
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 2d04435..82eb891 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c
@@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load_vmcs(vcpu, cpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); } void vmx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); vmx_prepare_switch_to_host(to_vmx(vcpu)); } @@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, * so that guest userspace can't DoS the guest simply by triggering * emulation (enclaves are CPL3 only). */ - if (vmx_get_exit_reason(vcpu).enclave_mode) { + if (vt_get_exit_reason(vcpu).enclave_mode) { kvm_queue_exception(vcpu, UD_VECTOR); return X86EMUL_PROPAGATE_FAULT; } @@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, static int skip_emulated_instruction(struct kvm_vcpu *vcpu) { - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); unsigned long rip, orig_rip; u32 instr_len; @@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) return skip_emulated_instruction(vcpu); } -static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +void vmx_clear_hlt(struct kvm_vcpu *vcpu) { /* * Ensure that we clear the HLT state in the VMCS. We don't need to @@ -2843,6 +2843,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, if (!cpu_has_sgx()) _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; + /* + * KVM doesn't re-derive the TSC scaling ratio when the host TSC + * frequency changes, so TSC scaling is only usable with a constant + * TSC. Clear the control here rather than in vmx_hardware_setup() so + * that the per-CPU configs recomputed by vmx_check_processor_compat() + * stay consistent with the golden vmcs_config. + */ + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) + _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING; + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) _cpu_based_3rd_exec_control = adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, @@ -3379,7 +3389,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) vpid_sync_context(vmx_get_current_vpid(vcpu)); } -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in @@ -4426,7 +4436,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (!vcpu->arch.apic->apicv_active) return -1; - __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); return 0; } @@ -4883,10 +4893,11 @@ int vmx_vcpu_precreate(struct kvm *kvm) #define VMX_XSS_EXIT_BITMAP 0 -static void init_vmcs(struct vcpu_vmx *vmx) +static void init_vmcs(struct kvm_vcpu *vcpu) { - struct kvm *kvm = vmx->vcpu.kvm; + struct kvm *kvm = vcpu->kvm; struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); + struct vcpu_vmx *vmx = to_vmx(vcpu); if (nested) nested_vmx_set_vmcs_shadowing_bitmap(); @@ -4911,7 +4922,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_tertiary_exec_ctrls()) tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); - if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { + if (enable_apicv && lapic_in_kernel(vcpu)) { vmcs_write64(EOI_EXIT_BITMAP0, 0); vmcs_write64(EOI_EXIT_BITMAP1, 0); vmcs_write64(EOI_EXIT_BITMAP2, 0); @@ -4923,7 +4934,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); } - if (vmx_can_use_ipiv(&vmx->vcpu)) { + if (vmx_can_use_ipiv(vcpu)) { vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); } @@ -4958,15 +4969,15 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) - vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); + vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); /* 22.2.1, 20.8.1 */ vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); - vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); - vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); + vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); + vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); set_cr4_guest_host_mask(vmx); @@ -4977,11 +4988,11 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); + vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page)); vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } - vmx_write_encls_bitmap(&vmx->vcpu, NULL); + vmx_write_encls_bitmap(vcpu, NULL); if (vmx_pt_mode_is_host_guest()) { memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); @@ -4994,13 +5005,13 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_writel(GUEST_SYSENTER_ESP, 0); vmcs_writel(GUEST_SYSENTER_EIP, 0); - vmx_guest_debugctl_write(&vmx->vcpu, 0); + vmx_guest_debugctl_write(vcpu, 0); if (cpu_has_vmx_tpr_shadow()) { vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (cpu_need_tpr_shadow(&vmx->vcpu)) + if (cpu_need_tpr_shadow(vcpu)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - __pa(vmx->vcpu.arch.apic->regs)); + __pa(vcpu->arch.apic->regs)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -5011,7 +5022,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - init_vmcs(vmx); + init_vmcs(vcpu); if (nested && kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) @@ -5381,15 +5392,9 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); } -static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->xfd && - !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); -} - static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { - unsigned long cr2 = vmx_get_exit_qual(vcpu); + unsigned long cr2 = vt_get_exit_qual(vcpu); if (vcpu->arch.apf.host_apf_flags) goto handle_pf; @@ -5437,7 +5442,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) u32 vect_info; vect_info = vmx->idt_vectoring_info; - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -5518,7 +5523,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) switch (ex_no) { case DB_VECTOR: - dr6 = vmx_get_exit_qual(vcpu); + dr6 = vt_get_exit_qual(vcpu); if (!(vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { /* @@ -5594,7 +5599,7 @@ static int handle_io(struct kvm_vcpu *vcpu) int size, in, string; unsigned port; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); string = (exit_qualification & 16) != 0; ++vcpu->stat.io_exits; @@ -5683,7 +5688,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int err; int ret; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); cr = exit_qualification & 15; reg = (exit_qualification >> 8) & 15; switch ((exit_qualification >> 4) & 3) { @@ -5761,7 +5766,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) int dr, dr7, reg; int err = 1; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); dr = exit_qualification & DEBUG_REG_ACCESS_NUM; /* First, if DR does not exist, trigger UD */ @@ -5857,7 +5862,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_invlpg(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); kvm_mmu_invlpg(vcpu, exit_qualification); return kvm_skip_emulated_instruction(vcpu); @@ -5866,7 +5871,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { if (likely(fasteoi)) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int access_type, offset; access_type = exit_qualification & APIC_ACCESS_TYPE; @@ -5887,17 +5892,16 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); - int vector = exit_qualification & 0xff; + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ - kvm_apic_set_eoi_accelerated(vcpu, vector); + kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); return 1; } static int handle_apic_write(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and @@ -5925,7 +5929,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); reason = (u32)exit_qualification >> 30; if (reason == TASK_SWITCH_GATE && idt_v) { @@ -5971,7 +5975,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) static int handle_ept_violation(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); gpa_t gpa; /* @@ -5999,7 +6003,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) return kvm_emulate_instruction(vcpu, 0); - return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification); + return __vt_handle_ept_violation(vcpu, gpa, exit_qualification); } static int handle_ept_misconfig(struct kvm_vcpu *vcpu) @@ -6162,7 +6166,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; @@ -6176,7 +6180,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) trace_kvm_pml_full(vcpu->vcpu_id); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); /* * PML buffer FULL happened while executing iret from NMI, @@ -6267,23 +6271,9 @@ static int handle_encls(struct kvm_vcpu *vcpu) } #endif /* CONFIG_X86_SGX_KVM */ -static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling to - * vmx_handle_exit(). - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - static int handle_notify(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); - bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; - - ++vcpu->stat.notify_window_exits; + unsigned long exit_qual = vt_get_exit_qual(vcpu); /* * Notify VM exit happened while executing iret from NMI, @@ -6293,15 +6283,7 @@ static int handle_notify(struct kvm_vcpu *vcpu) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); - if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || - context_invalid) { - vcpu->run->exit_reason = KVM_EXIT_NOTIFY; - vcpu->run->notify.flags = context_invalid ? - KVM_NOTIFY_CONTEXT_INVALID : 0; - return 0; - } - - return 1; + return __vt_handle_notify(vcpu, exit_qual); } static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) @@ -6311,13 +6293,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } @@ -6377,7 +6359,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_VMFUNC] = handle_vmx_instruction, [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, [EXIT_REASON_ENCLS] = handle_encls, - [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, + [EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit, [EXIT_REASON_NOTIFY] = handle_notify, [EXIT_REASON_SEAMCALL] = handle_tdx_instruction, [EXIT_REASON_TDCALL] = handle_tdx_instruction, @@ -6388,16 +6370,18 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers)); + void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); *reason = vmx->vt.exit_reason.full; - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); if (!(vmx->vt.exit_reason.failed_vmentry)) { *info2 = vmx->idt_vectoring_info; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(*intr_info)) *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); else @@ -6418,48 +6402,25 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) *error_code = 0; } -static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) +static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) { - if (vmx->pml_pg) { - __free_page(vmx->pml_pg); - vmx->pml_pg = NULL; + if (vcpu->arch.pml_page) { + __free_page(vcpu->arch.pml_page); + vcpu->arch.pml_page = NULL; } } static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); - u16 pml_idx, pml_tail_index; - u64 *pml_buf; - int i; + u16 pml_idx; pml_idx = vmcs_read16(GUEST_PML_INDEX); /* Do nothing if PML buffer is empty */ if (pml_idx == PML_HEAD_INDEX) return; - /* - * PML index always points to the next available PML buffer entity - * unless PML log has just overflowed. - */ - pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; - /* - * PML log is written backwards: the CPU first writes the entry 511 - * then the entry 510, and so on. - * - * Read the entries in the same order they were written, to ensure that - * the dirty ring is filled in the same order the CPU wrote them. - */ - pml_buf = page_address(vmx->pml_pg); - - for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { - u64 gpa; - - gpa = pml_buf[i]; - WARN_ON(gpa & (PAGE_SIZE - 1)); - kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); - } + kvm_flush_pml_buffer(vcpu, pml_idx); /* reset PML index */ vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); @@ -6705,10 +6666,10 @@ void dump_vmcs(struct kvm_vcpu *vcpu) * The guest has exited. See if we can fix it or if we need userspace * assistance. */ -static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); u32 vectoring_info = vmx->idt_vectoring_info; u16 exit_handler_index; @@ -6864,24 +6825,6 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return 0; } -int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) -{ - int ret = __vmx_handle_exit(vcpu, exit_fastpath); - - /* - * Exit to user space when bus lock detected to inform that there is - * a bus lock in guest. - */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - return 0; - } - return ret; -} - void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { int tpr_threshold; @@ -7145,74 +7088,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); } -static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) -{ - /* - * Save xfd_err to guest_fpu before interrupt is enabled, so the - * MSR value is not clobbered by the host activity before the guest - * has chance to consume it. - * - * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM - * interception may have been caused by L1 interception. Per the SDM, - * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. - * - * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. - * unlike CR2 and DR6, the value is not a payload that is attached to - * the #NM exception. - */ - if (is_xfd_nm_fault(vcpu)) - rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); -} - -static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) -{ - /* if exit due to PF check for async PF */ - if (is_page_fault(intr_info)) - vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - /* if exit due to NM, handle before interrupts are enabled */ - else if (is_nm_fault(intr_info)) - handle_nm_fault_irqoff(vcpu); - /* Handle machine checks before interrupts are enabled */ - else if (is_machine_check(intr_info)) - kvm_machine_check(); -} - -static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, - u32 intr_info) -{ - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - - if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, - "unexpected VM-Exit interrupt info: 0x%x", intr_info)) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); - kvm_after_interrupt(vcpu); - - vcpu->arch.at_instruction_boundary = true; -} - -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) -{ - if (to_vt(vcpu)->emulation_required) - return; - - switch (vmx_get_exit_reason(vcpu).basic) { - case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_MCE_DURING_VMENTRY: - kvm_machine_check(); - break; - default: - break; - } -} - /* * The kvm parameter can be NULL (module initialization, or invocation before * VM creation). Be sure to check the kvm parameter before using it. @@ -7252,7 +7127,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) if (vmx->loaded_vmcs->nmi_known_unmasked) return; - exit_intr_info = vmx_get_intr_info(&vmx->vcpu); + exit_intr_info = vt_get_intr_info(&vmx->vcpu); unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; vector = exit_intr_info & INTR_INFO_VECTOR_MASK; /* @@ -7421,14 +7296,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, * the fastpath even, all other exits must use the slow path. */ if (is_guest_mode(vcpu) && - vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) + vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) return EXIT_FASTPATH_NONE; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_wrmsr(vcpu); case EXIT_REASON_MSR_WRITE_IMM: - return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); case EXIT_REASON_PREEMPTION_TIMER: return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); @@ -7441,17 +7316,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, } } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) -{ - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); - x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); - kvm_after_interrupt(vcpu); -} - static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned int flags) { @@ -7476,15 +7340,15 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vmx_enable_fb_clear(vmx); if (unlikely(vmx->fail)) { - vmx->vt.exit_reason.full = 0xdead; + vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED; goto out; } vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); - if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) + if (likely(!vt_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); out: guest_state_exit_irqoff(); @@ -7621,7 +7485,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * checking. */ if (vcpu->arch.nested_run_pending && - !vmx_get_exit_reason(vcpu).failed_vmentry) + !vt_get_exit_reason(vcpu).failed_vmentry) ++vcpu->stat.nested_run; vcpu->arch.nested_run_pending = 0; @@ -7632,7 +7496,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) trace_kvm_exit(vcpu, KVM_ISA_VMX); - if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry)) + if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; @@ -7650,7 +7514,7 @@ void vmx_vcpu_free(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); if (enable_pml) - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid(vmx->vpid); nested_vmx_free_vcpu(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); @@ -7682,8 +7546,8 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) * for the guest), etc. */ if (enable_pml) { - vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!vmx->pml_pg) + vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!vcpu->arch.pml_page) goto free_vpid; } @@ -7754,7 +7618,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) free_vmcs: free_loaded_vmcs(vmx->loaded_vmcs); free_pml: - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid: free_vpid(vmx->vpid); return err; @@ -8139,6 +8003,13 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_IBT); } + /* + * CPUID 0x80000008. Tell userspace that EFER.LMSLE must be zero; KVM + * never allows EFER.LMSLE to be set on Intel CPUs, as KVM supports long + * mode segment limits only in conjunction with nested SVM. + */ + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } @@ -8442,21 +8313,13 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit } #endif -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!enable_pml)) - return; - guard(vmx_vmcs01)(vcpu); - /* - * Note, nr_memslots_dirty_logging can be changed concurrent with this - * code, but in that case another update request will be made and so - * the guest will never run with a stale PML value. - */ - if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) + if (enable) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); else secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML);
diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 1db4610..74cd50f 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h
@@ -265,13 +265,6 @@ struct vcpu_vmx { unsigned int ple_window; bool ple_window_dirty; - /* Support for PML */ -#define PML_LOG_NR_ENTRIES 512 - /* PML is written backwards: this is the first entry written by the CPU */ -#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) - - struct page *pml_pg; - /* apic deadline value in host tsc */ u64 hv_deadline_tsc; @@ -304,43 +297,6 @@ struct kvm_vmx { u64 *pid_table; }; -static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) -{ - return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt); -} - -static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) -{ - return &(container_of(vt, struct vcpu_vmx, vt)->vcpu); -} - -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) -{ - return to_vt(vcpu)->exit_reason; -} - -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - - return vt->exit_qualification; -} - -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - return vt->exit_intr_info; -} - void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); @@ -397,7 +353,7 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated); bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated);
diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14..cd72684 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h
@@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_put(struct kvm_vcpu *vcpu); int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath); -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu); int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu); void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu); bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu); @@ -82,7 +81,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); bool vmx_get_if_flag(struct kvm_vcpu *vcpu); void vmx_flush_tlb_all(struct kvm_vcpu *vcpu); void vmx_flush_tlb_current(struct kvm_vcpu *vcpu); -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full); +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr); void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); @@ -95,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection); int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection); bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); +void vmx_clear_hlt(struct kvm_vcpu *vcpu); void vmx_enable_nmi_window(struct kvm_vcpu *vcpu); void vmx_enable_irq_window(struct kvm_vcpu *vcpu); void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr); @@ -118,7 +118,7 @@ u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); void vmx_write_tsc_offset(struct kvm_vcpu *vcpu); void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); #ifdef CONFIG_X86_64 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index aad065d..32563a9 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c
@@ -35,6 +35,7 @@ #include "smm.h" #include <linux/clocksource.h> +#include <linux/timekeeping.h> #include <linux/interrupt.h> #include <linux/kvm.h> #include <linux/fs.h> @@ -61,6 +62,7 @@ #include <linux/mem_encrypt.h> #include <linux/suspend.h> #include <linux/smp.h> +#include <linux/units.h> #include <trace/events/ipi.h> #include <trace/events/kvm.h> @@ -399,30 +401,37 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, switch (ex->vector) { case DB_VECTOR: /* - * "Certain debug exceptions may clear bit 0-3. The - * remaining contents of the DR6 register are never - * cleared by the processor". + * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so + * to maintain backwards compatibility with existing software, + * features that use previously-reserved bits have active-low + * semantics, i.e. clear the bit when the feature is present in + * the payload. + * + * Further complicating matters, some DR6 bits are preserved by + * hardware, while others are explicitly modified on every #DB. + * The trap bits are always set based on the payload, as is the + * RTM flag (but it's active low). All other bits are modified + * if and only if a relevant debug exception occurs, e.g. BD, + * BS, and BT are never cleared by hardware, and BLD is never + * set by hardware (when supported, excepting RESET). + * + * Lastly, the payload does NOT have active-low semantics, e.g. + * so that it's compatible VMX's pending debug exceptions and + * qualification fields, and to avoid bleeding the DR6 madness + * into other KVM code. + * + * To compute DR6: + * + * 1. "Reset" the bits that are modified on all #DBs + * 2. Clear active-low bits that are present in the payload. + * 3. Set active-high bits that are present in the payload. + * 4. Clear fixed-0 bits. + * 5. Set fixed-1 bits. */ vcpu->arch.dr6 &= ~DR_TRAP_BITS; - /* - * In order to reflect the #DB exception payload in guest - * dr6, three components need to be considered: active low - * bit, FIXED_1 bits and active high bits (e.g. DR6_BD, - * DR6_BS and DR6_BT) - * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits. - * In the target guest dr6: - * FIXED_1 bits should always be set. - * Active low bits should be cleared if 1-setting in payload. - * Active high bits should be set if 1-setting in payload. - * - * Note, the payload is compatible with the pending debug - * exceptions/exit qualification under VMX, that active_low bits - * are active high in payload. - * So they need to be flipped for DR6. - */ - vcpu->arch.dr6 |= DR6_ACTIVE_LOW; - vcpu->arch.dr6 |= ex->payload; - vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; + vcpu->arch.dr6 |= DR6_RTM; + vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); + vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); /* * The #DB payload is defined as compatible with the 'pending @@ -431,6 +440,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; @@ -448,6 +458,8 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto { struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + kvm_make_request(KVM_REQ_EVENT, vcpu); + ex->vector = vector; ex->injected = false; ex->pending = true; @@ -925,19 +937,13 @@ static void update_pvclock_gtod(struct timekeeper *tk) write_seqcount_end(&vdata->seq); } +#endif static s64 get_kvmclock_base_ns(void) { /* Count up from boot time, but with the frequency of the raw clock. */ - return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot)); + return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); } -#else -static s64 get_kvmclock_base_ns(void) -{ - /* Master clock not used, so we can just use CLOCK_BOOTTIME. */ - return ktime_get_boottime_ns(); -} -#endif static uint32_t div_frac(uint32_t dividend, uint32_t divisor) { @@ -945,32 +951,57 @@ static uint32_t div_frac(uint32_t dividend, uint32_t divisor) return dividend; } -static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz, +static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz, s8 *pshift, u32 *pmultiplier) { - uint64_t scaled64; - int32_t shift = 0; - uint64_t tps64; - uint32_t tps32; + u64 scaled_hz_u64 = scaled_hz; + s32 shift = 0; + u64 base_hz_u64; + u32 base32; - tps64 = base_hz; - scaled64 = scaled_hz; - while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { - tps64 >>= 1; + /* + * This function calculates a fixed-point multiplier and shift such + * that: + * time_ns = (tsc_cycles << shift) * multiplier >> 32 + * + * Where tsc_cycles tick at base_hz, and time_ns should count at + * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion). + * + * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift + * to keep everything in range. + */ + + base_hz_u64 = base_hz; + + /* + * Start by shifting base_hz right until it fits in 32 bits, and + * is lower than double the target rate. This introduces a negative + * shift value which would result in pvclock_scale_delta() shifting + * the actual tick count right before performing the multiplication. + */ + while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) { + base_hz_u64 >>= 1; shift--; } - tps32 = (uint32_t)tps64; - while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { - if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) - scaled64 >>= 1; + /* Now the shifted base_hz fits in 32 bits. */ + base32 = (u32)base_hz_u64; + + /* + * Next, shift scaled_hz right until it fits in 32 bits, and ensure + * that the shifted base_hz is strictly larger (so that the result of the + * final division also fits in 32 bits). + */ + while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) { + if (scaled_hz_u64 >> 32 || base32 & BIT(31)) + scaled_hz_u64 >>= 1; else - tps32 <<= 1; + base32 <<= 1; shift++; } *pshift = shift; - *pmultiplier = div_frac(scaled64, tps32); + *pmultiplier = div_frac(scaled_hz_u64, base32); } #ifdef CONFIG_X86_64 @@ -1061,11 +1092,15 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) { - u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, - vcpu->arch.virtual_tsc_mult, - vcpu->arch.virtual_tsc_shift); - tsc += vcpu->arch.this_tsc_write; - return tsc; + s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec; + u64 tsc; + + /* Handle negative deltas gracefully (master clock ref may be earlier) */ + tsc = pvclock_scale_delta(abs(delta_ns), + vcpu->arch.virtual_tsc_mult, + vcpu->arch.virtual_tsc_shift); + + return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc); } #ifdef CONFIG_X86_64 @@ -1131,11 +1166,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio) return _tsc; } -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc) { u64 tsc; - tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); + tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); return target_tsc - tsc; } @@ -1164,11 +1200,58 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) { - if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio) - return mul_u64_u64_shr(l1_multiplier, l2_multiplier, - kvm_caps.tsc_scaling_ratio_frac_bits); + u8 frac_bits = kvm_caps.tsc_scaling_ratio_frac_bits; + u64 nested_multiplier; - return l1_multiplier; + if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) + return l1_multiplier; + + /* + * The shift is fixed on both AMD and Intel, and operates on a 64-bit + * value. I.e. a shift greater than 63 is completely nonsensical. + */ + if (WARN_ON_ONCE(frac_bits > 63)) + return l1_multiplier; + + /* + * If the resulting multiplier can't be programmed into hardware, run + * L2 at the minimum/maximum frequency supported by hardware, i.e. + * saturate L2's frequency on both sides. Because L2's frequency needs + * to be distilled down to a single multiplier to get from: + * + * L2 = (((L0 * L1_mult) >> frac) * L2_mult) >> frac) + * + * to: + * + * L2 = (L0 * mult) >> frac + * + * very small/large L1 and L2 multipliers can underflow/overflow the + * minimum/maximum multiplier supported by hardware when combined into + * a single value. + * + * Manually check for the case where the result would overflow a u64, + * i.e. if the multiplier would be silently truncated before the "too + * large" check. Avoid doing the multiply twice in the common case + * where the compiler natively supports 128-bit values. + */ +#ifdef CONFIG_ARCH_SUPPORTS_INT128 + unsigned __int128 m = (unsigned __int128)l1_multiplier * l2_multiplier; + + if (m >> (64 + frac_bits)) + return kvm_caps.max_tsc_scaling_ratio; + + nested_multiplier = m >> frac_bits; +#else + if (mul_u64_u64_shr(l1_multiplier, l2_multiplier, 64 + frac_bits)) + return kvm_caps.max_tsc_scaling_ratio; + + nested_multiplier = mul_u64_u64_shr(l1_multiplier, l2_multiplier, frac_bits); +#endif + if (nested_multiplier > kvm_caps.max_tsc_scaling_ratio) + return kvm_caps.max_tsc_scaling_ratio; + + /* The minimum multiplier is '1' on both AMD and Intel. */ + return nested_multiplier ?: 1; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier); @@ -1254,6 +1337,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, kvm->arch.last_tsc_write = tsc; kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; kvm->arch.last_tsc_offset = offset; + kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; vcpu->arch.last_guest_tsc = tsc; @@ -1296,7 +1380,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) bool synchronizing = false; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); ns = get_kvmclock_base_ns(); elapsed = ns - kvm->arch.last_tsc_nsec; @@ -1345,7 +1429,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) } else { u64 delta = nsec_to_cycles(vcpu, elapsed); data += delta; - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); } matched = true; } @@ -1356,126 +1440,27 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) #ifdef CONFIG_X86_64 -static u64 read_tsc(void) +static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, + u64 *tsc_timestamp) { - u64 ret = (u64)rdtsc_ordered(); - u64 last = pvclock_gtod_data.clock.cycle_last; - - if (likely(ret >= last)) - return ret; - /* - * GCC likes to generate cmov here, but this branch is extremely - * predictable (it's just a function of time and the likely is - * very likely) and there's a data dependence, so force GCC - * to generate a branch instead. I don't barrier() because - * we don't actually need a barrier, and if this function - * ever gets inlined it will generate worse code. + * ktime_get_snapshot_id() cannot fail for standard clock IDs + * (only for invalid/aux clocks or during suspend, with a WARN). */ - asm volatile (""); - return last; -} + if (!snap->valid) + return false; -static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, - int *mode) -{ - u64 tsc_pg_val; - long v; - - switch (clock->vclock_mode) { - case VDSO_CLOCKMODE_HVCLOCK: - if (hv_read_tsc_page_tsc(hv_get_tsc_page(), - tsc_timestamp, &tsc_pg_val)) { - /* TSC page valid */ - *mode = VDSO_CLOCKMODE_HVCLOCK; - v = (tsc_pg_val - clock->cycle_last) & - clock->mask; - } else { - /* TSC page invalid */ - *mode = VDSO_CLOCKMODE_NONE; - } - break; - case VDSO_CLOCKMODE_TSC: - *mode = VDSO_CLOCKMODE_TSC; - *tsc_timestamp = read_tsc(); - v = (*tsc_timestamp - clock->cycle_last) & - clock->mask; - break; - default: - *mode = VDSO_CLOCKMODE_NONE; + if (snap->cs_id == CSID_X86_TSC) { + *tsc_timestamp = snap->cycles; + return true; } - if (*mode == VDSO_CLOCKMODE_NONE) - *tsc_timestamp = v = 0; + if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) { + *tsc_timestamp = snap->hw_cycles; + return true; + } - return v * clock->mult; -} - -/* - * As with get_kvmclock_base_ns(), this counts from boot time, at the - * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot). - */ -static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->raw_clock.base_cycles; - ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode); - ns >>= gtod->raw_clock.shift; - ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot)); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - -/* - * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with - * no boot time offset. - */ -static int do_monotonic(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - ns += ktime_to_ns(gtod->clock.offset); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - -static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ts->tv_sec = gtod->wall_time_sec; - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return mode; + return false; } /* @@ -1485,12 +1470,14 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) */ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT)); + return true; } /* @@ -1499,12 +1486,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) */ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_monotonic(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(snap.systime); + return true; } /* @@ -1517,11 +1506,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_REALTIME, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp)); + *ts = ktime_to_timespec64(snap.systime); + return true; } #endif @@ -1566,6 +1558,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, * */ +static unsigned long get_cpu_tsc_khz(void); + static void pvclock_update_vm_gtod_copy(struct kvm *kvm) { #ifdef CONFIG_X86_64 @@ -1589,9 +1583,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm) && !ka->backwards_tsc_observed && !ka->boot_vcpu_runs_old_kvmclock; - if (ka->use_master_clock) + if (ka->use_master_clock) { + u64 tsc_hz; + atomic_set(&kvm_guest_has_master_clock, 1); + /* + * Copy the scaling ratio and precompute the mul/shift for + * converting guest TSC to nanoseconds. These are used by + * get_kvmclock() to compute kvmclock from the host TSC + * without needing a vCPU reference. + */ + ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio; + tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (tsc_hz && kvm_caps.has_tsc_control) + tsc_hz = kvm_scale_tsc(tsc_hz, + ka->master_tsc_scaling_ratio); + if (tsc_hz) + kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, + &ka->master_tsc_shift, + &ka->master_tsc_mul); + else + ka->use_master_clock = false; + } + vclock_mode = pvclock_gtod_data.clock.vclock_mode; trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, vcpus_matched); @@ -1658,39 +1673,49 @@ static unsigned long get_cpu_tsc_khz(void) } /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ -static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) +static bool __get_kvmclock_master_clock(struct kvm *kvm, + struct kvm_clock_data *data) { +#ifdef CONFIG_X86_64 struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; + struct timespec64 ts; - /* both __this_cpu_read() and rdtsc() should be on the same cpu */ - get_cpu(); + if (!ka->use_master_clock) + return false; - data->flags = 0; - if (ka->use_master_clock && - (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) { -#ifdef CONFIG_X86_64 - struct timespec64 ts; + if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) + return false; - if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { - data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; - data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; - } else -#endif - data->host_tsc = rdtsc(); + data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; + data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC | + KVM_CLOCK_TSC_STABLE; - data->flags |= KVM_CLOCK_TSC_STABLE; - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); + hv_clock.tsc_timestamp = ka->master_cycle_now; + hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; + + /* + * Use the precomputed guest-TSC-based mul/shift so that the kvmclock + * value matches what the guest computes from its own TSC. + */ + hv_clock.tsc_shift = ka->master_tsc_shift; + hv_clock.tsc_to_system_mul = ka->master_tsc_mul; + + if (kvm_caps.has_tsc_control) { + u64 tsc_delta = data->host_tsc - ka->master_cycle_now; + + tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio); + data->clock = hv_clock.system_time + + pvclock_scale_delta(tsc_delta, + hv_clock.tsc_to_system_mul, + hv_clock.tsc_shift); } else { - data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; + data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); } - - put_cpu(); + return true; +#else + return false; +#endif } static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) @@ -1699,8 +1724,11 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) unsigned seq; do { + data->flags = 0; + seq = read_seqcount_begin(&ka->pvclock_sc); - __get_kvmclock(kvm, data); + if (!__get_kvmclock_master_clock(kvm, data)) + data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; } while (read_seqcount_retry(&ka->pvclock_sc, seq)); } @@ -1762,36 +1790,45 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { + u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns; + struct kvm_arch *ka __maybe_unused = &v->kvm->arch; struct pvclock_vcpu_time_info hv_clock = {}; - unsigned long flags, tgt_tsc_khz; - unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; - struct kvm_arch *ka = &v->kvm->arch; s64 kernel_ns; - u64 tsc_timestamp, host_tsc; - bool use_master_clock; - - kernel_ns = 0; - host_tsc = 0; /* * If the host uses TSC clock, then passthrough TSC as stable * to the guest. */ +#ifdef CONFIG_X86_64 + bool use_master_clock; + unsigned int seq; + do { seq = read_seqcount_begin(&ka->pvclock_sc); use_master_clock = ka->use_master_clock; - if (use_master_clock) { - host_tsc = ka->master_cycle_now; - kernel_ns = ka->master_kernel_ns; - } - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + if (!use_master_clock) + continue; - /* Keep irq disabled to prevent changes to the clock */ - local_irq_save(flags); - tgt_tsc_khz = get_cpu_tsc_khz(); - if (unlikely(tgt_tsc_khz == 0)) { - local_irq_restore(flags); + if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) { + use_master_clock = false; + continue; + } + + master_tsc = ka->master_cycle_now; + master_ns = ka->master_kernel_ns; + } while (read_seqcount_retry(&ka->pvclock_sc, seq)); +#else + const bool use_master_clock = false; +#endif + /* + * Ensure reading the TSC+frequency pair is done on the same CPU. When + * NOT using the master clock, the TSC frequency may vary between CPUs. + */ + preempt_disable(); + tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (unlikely(tgt_tsc_hz == 0)) { + preempt_enable(); kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); return 1; } @@ -1820,28 +1857,43 @@ int kvm_guest_time_update(struct kvm_vcpu *v) } } - local_irq_restore(flags); + /* + * Refresh L1's last "observed" TSC to match the PV clock's timestamp, + * e.g. so that the guest can't see a TSC that's behind the reference. + */ + vcpu->last_guest_tsc = tsc_timestamp; + + preempt_enable(); /* With all the info we got, fill in the values */ if (kvm_caps.has_tsc_control) { - tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz, + tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz, v->arch.l1_tsc_scaling_ratio); - tgt_tsc_khz = tgt_tsc_khz ? : 1; + tgt_tsc_hz = tgt_tsc_hz ? : 1; } - if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { - kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL, + if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) { + kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz, &vcpu->pvclock_tsc_shift, &vcpu->pvclock_tsc_mul); - vcpu->hw_tsc_khz = tgt_tsc_khz; + vcpu->hw_tsc_hz = tgt_tsc_hz; } hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; - hv_clock.tsc_timestamp = tsc_timestamp; - hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; - vcpu->last_guest_tsc = tsc_timestamp; + /* + * If the master clock is NOT in use, the reference time placed in the + * hv_clock is "now". If master clock is in use, the reference time is + * the master clock's snapshot from some time in the past, not "now". + */ + if (use_master_clock) { + hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc); + hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset; + } else { + hv_clock.tsc_timestamp = tsc_timestamp; + hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; + } /* If the host uses TSC clocksource, then it is stable */ hv_clock.flags = 0; @@ -1903,63 +1955,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v) * wallclock and kvmclock times, and subtracting one from the other. * * Fall back to using their values at slightly different moments by - * calling ktime_get_real_ns() and get_kvmclock_ns() separately. + * calling ktime_get_real_ns() and get_kvmclock() separately. */ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) { -#ifdef CONFIG_X86_64 - struct pvclock_vcpu_time_info hv_clock; - struct kvm_arch *ka = &kvm->arch; - unsigned long seq, local_tsc_khz; - struct timespec64 ts; - uint64_t host_tsc; + struct kvm_clock_data data; - do { - seq = read_seqcount_begin(&ka->pvclock_sc); - - local_tsc_khz = 0; - if (!ka->use_master_clock) - break; - - /* - * The TSC read and the call to get_cpu_tsc_khz() must happen - * on the same CPU. - */ - get_cpu(); - - local_tsc_khz = get_cpu_tsc_khz(); - - if (local_tsc_khz && - !kvm_get_walltime_and_clockread(&ts, &host_tsc)) - local_tsc_khz = 0; /* Fall back to old method */ - - put_cpu(); - - /* - * These values must be snapshotted within the seqcount loop. - * After that, it's just mathematics which can happen on any - * CPU at any time. - */ - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + get_kvmclock(kvm, &data); /* - * If the conditions were right, and obtaining the wallclock+TSC was - * successful, calculate the KVM clock at the corresponding time and - * subtract one from the other to get the guest's epoch in nanoseconds - * since 1970-01-01. + * If get_kvmclock() captured both wallclock and kvmclock from the + * same TSC reading, use them for a precise epoch calculation. */ - if (local_tsc_khz) { - kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec - - __pvclock_read_cycles(&hv_clock, host_tsc); - } -#endif - return ktime_get_real_ns() - get_kvmclock_ns(kvm); + if (data.flags & KVM_CLOCK_REALTIME) + return data.realtime - data.clock; + + return ktime_get_real_ns() - data.clock; } /* @@ -2583,7 +2594,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) mark_tsc_unstable("KVM discovered backwards TSC"); if (kvm_check_tsc_unstable()) { - u64 offset = kvm_compute_l1_tsc_offset(vcpu, + u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), vcpu->arch.last_guest_tsc); kvm_vcpu_write_tsc_offset(vcpu, offset); if (!vcpu->arch.guest_tsc_protected) @@ -3717,7 +3728,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, user_tsc_khz = (u32)arg; if (kvm_caps.has_tsc_control && - user_tsc_khz >= kvm_caps.max_guest_tsc_khz) + user_tsc_khz > kvm_caps.max_guest_tsc_khz) goto out; if (user_tsc_khz == 0) @@ -3941,6 +3952,37 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) kvm_vcpu_kick(vcpu); } +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) +{ + u16 pml_tail_index; + u64 *pml_buf; + int i; + + /* + * PML index always points to the next available PML buffer entity + * unless PML log has just overflowed. + */ + pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; + + /* + * PML log is written backwards: the CPU first writes the entry 511 + * then the entry 510, and so on. + * + * Read the entries in the same order they were written, to ensure that + * the dirty ring is filled in the same order the CPU wrote them. + */ + pml_buf = page_address(vcpu->arch.pml_page); + + for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { + u64 gpa; + + gpa = pml_buf[i]; + WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); + kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); + int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { @@ -4653,7 +4695,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) user_tsc_khz = (u32)arg; if (kvm_caps.has_tsc_control && - user_tsc_khz >= kvm_caps.max_guest_tsc_khz) + user_tsc_khz > kvm_caps.max_guest_tsc_khz) goto out; if (user_tsc_khz == 0) @@ -5089,7 +5131,8 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, * operation using rep will only have the initial GPA from the NPF * occurred. */ - if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && + if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && + emulator_can_use_gpa(ctxt) && (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { gpa = ctxt->gpa_val; ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); @@ -5938,7 +5981,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - ctxt->gpa_available = false; + ctxt->gpa_access = 0; ctxt->eflags = kvm_get_rflags(vcpu); ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; @@ -6454,7 +6497,17 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, /* With shadow page tables, cr2 contains a GVA or nGPA. */ if (vcpu->arch.mmu->root_role.direct) { - ctxt->gpa_available = true; + ctxt->gpa_access = ACC_READ_MASK; + /* + * Always allow writes for guests with protected page + * tables, as KVM can't walk the guest's page tables, + * i.e. KVM can't get the RMW protections for a given + * GVA to see if the write side of a RMW operation + * should be allowed. + */ + if ((emulation_type & EMULTYPE_PF_WRITE) || + vcpu->kvm->arch.has_protected_page_tables) + ctxt->gpa_access |= ACC_WRITE_MASK; ctxt->gpa_val = cr2_or_gpa; } } else { @@ -6930,7 +6983,14 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { kvm_caps.supported_efer_bits |= EFER_SVME; - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + + /* + * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 + * would be nonsensical. Note, vendor code sets the defeature + * if KVM can't support EFER.LMSLE for any reason, i.e. this + * needs to consult KVM's capabilities, not just raw CPUID. + */ + if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) kvm_caps.supported_efer_bits |= EFER_LMSLE; } } @@ -7131,7 +7191,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) if (kvm_caps.has_tsc_control) { /* * Make sure the user can only configure tsc_khz values that - * fit into a signed integer. + * fit into a signed integer, otherwise KVM_GET_TSC_KHZ would + * return a negative value and confuse userspace. * A min value is not calculated because it will always * be 1 on all machines. */ @@ -7925,10 +7986,8 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, kvm->arch.apicv_inhibit_reasons = new; if (new) { unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); - int idx = srcu_read_lock(&kvm->srcu); kvm_zap_gfn_range(kvm, gfn, gfn+1); - srcu_read_unlock(&kvm->srcu, idx); } } else { kvm->arch.apicv_inhibit_reasons = new; @@ -8042,6 +8101,21 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) kvm_x86_call(set_apic_access_page_addr)(vcpu); } +static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +{ + /* + * Note, nr_memslots_dirty_logging can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); + + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) + return; + + kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); +} + /* * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without @@ -8155,7 +8229,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_NMI, vcpu)) process_nmi(vcpu); if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { - BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); if (test_bit(vcpu->arch.pending_ioapic_eoi, vcpu->arch.ioapic_handled_vectors)) { vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; @@ -8212,7 +8285,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(recalc_intercepts)(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) - kvm_x86_call(update_cpu_dirty_logging)(vcpu); + kvm_update_cpu_dirty_logging(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { kvm_vcpu_reset(vcpu, true); @@ -8401,6 +8474,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); kvm_x86_call(sync_dirty_debug_regs)(vcpu); kvm_update_dr0123(vcpu); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); kvm_update_dr7(vcpu); } @@ -9062,6 +9136,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, } kvm_set_mp_state(vcpu, mp_state->mp_state); + + /* + * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's + * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state + * other than HALTED. The hardware state is sticky across VM-Exit and + * VM-Enter and is not touched by any other ioctl, so a vCPU that halted + * with HLT-exiting disabled stays wedged even after userspace rewrites + * its registers, e.g. when a VMM emulates a machine reset. + */ + if (kvm_hlt_in_guest(vcpu->kvm) && + mp_state->mp_state != KVM_MP_STATE_HALTED) + kvm_x86_call(clear_hlt)(vcpu); + kvm_make_request(KVM_REQ_EVENT, vcpu); ret = 0; @@ -9444,6 +9531,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) return; vcpu_load(vcpu); kvm_synchronize_tsc(vcpu, NULL); + if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) + kvm_update_masterclock(vcpu->kvm); vcpu_put(vcpu); /* poll control enabled by default */ @@ -10144,7 +10233,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, } } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES kvm_mmu_init_memslot_memory_attributes(kvm, slot); #endif @@ -10641,14 +10730,15 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) } #ifdef CONFIG_KVM_GUEST_MEMFD -/* - * KVM doesn't yet support initializing guest_memfd memory as shared for VMs - * with private memory (the private vs. shared tracking needs to be moved into - * guest_memfd). - */ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) { - return !kvm_arch_has_private_mem(kvm); + /* + * INIT_SHARED is supported if in-place conversion is enabled, or if + * the VM doesn't support private memory. If the VM has private memory + * and in-place conversion is disabled, then guest_memfd can _only_ be + * used for private memory. + */ + return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm); } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT @@ -10657,6 +10747,11 @@ int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, { return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } + +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +{ + kvm_x86_call(gmem_make_shared)(pfn, nr_pages); +} #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index f72897a..436e2fb 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h
@@ -335,7 +335,8 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio); u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc); +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc); void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, @@ -407,6 +408,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case * the CR2/GPA value pass on the stack is valid. * + * EMULTYPE_PF_WRITE - Set with EMULTYPE_PF when hardware reports a write access. + * * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility * state and inject single-step #DBs after skipping * an instruction (after completing userspace I/O). @@ -445,6 +448,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) #define EMULTYPE_SKIP_SOFT_INT (1 << 9) +#define EMULTYPE_PF_WRITE (1 << 10) #define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16) #define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff) @@ -900,4 +904,11 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); +/* Support for PML */ +#define PML_LOG_NR_ENTRIES 512 +/* PML is written backwards: this is the first entry written by the CPU */ +#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) + +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx); + #endif
diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index d113bd0..6aa00f7 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c
@@ -73,7 +73,7 @@ static int kvm_xen_shared_info_init(struct kvm *kvm) BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00); BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c); - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; wc_sec_hi = &shinfo->wc_sec_hi; @@ -389,7 +389,7 @@ static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic) BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { user_len = sizeof(struct vcpu_runstate_info); times_ofs = offsetof(struct vcpu_runstate_info, state_entry_time); @@ -676,28 +676,32 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) } /* Now gpc->khva is a valid kernel address for the vcpu_info */ - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; + void *vi_pending_sel = &vi->evtchn_pending_sel; - asm volatile(LOCK_PREFIX "orq %0, %1\n" - "notq %0\n" - LOCK_PREFIX "andq %0, %2\n" - : "=r" (evtchn_pending_sel), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel)); + if (IS_ALIGNED((unsigned long)vi_pending_sel, sizeof(u64))) { + atomic64_or(evtchn_pending_sel, vi_pending_sel); + } else { + atomic_or(evtchn_pending_sel, vi_pending_sel); + /* + * The cast keeps the shift well-defined on 32-bit, + * where evtchn_pending_sel is 32 bits wide and this + * branch is unreachable anyway (this is inside + * kvm_xen_has_64bit_shinfo(), which is gated on + * IS_ENABLED(CONFIG_64BIT)). + */ + atomic_or((u64)evtchn_pending_sel >> 32, + vi_pending_sel + 4); + } + + atomic64_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } else { - u32 evtchn_pending_sel32 = evtchn_pending_sel; struct compat_vcpu_info *vi = gpc->khva; - asm volatile(LOCK_PREFIX "orl %0, %1\n" - "notl %0\n" - LOCK_PREFIX "andl %0, %2\n" - : "=r" (evtchn_pending_sel32), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel32)); + atomic_or(evtchn_pending_sel, (void *)&vi->evtchn_pending_sel); + atomic_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } @@ -728,6 +732,16 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) BUILD_BUG_ON(sizeof(rc) != sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending)); + /* + * kvm_gpc_check() checks the memslot generation, so kvm->srcu must be + * held. Most callers hold it already, but this is also reached from + * kvm_emulate_halt() on the VM-Exit path and from kvm_vcpu_block(), + * where vcpu_enter_guest() has already dropped the vCPU's SRCU lock. + * Taking SRCU does not sleep, so it is safe even in the atomic case + * which is handled below. + */ + guard(srcu)(&v->kvm->srcu); + read_lock_irqsave(&gpc->lock, flags); while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) { read_unlock_irqrestore(&gpc->lock, flags); @@ -941,6 +955,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.gpa, sizeof(u32))) + break; + r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache, data->u.gpa, sizeof(struct vcpu_info)); } else { @@ -950,6 +968,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.hva, sizeof(u32))) + break; + r = kvm_gpc_activate_hva(&vcpu->arch.xen.vcpu_info_cache, data->u.hva, sizeof(struct vcpu_info)); } @@ -993,7 +1015,7 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) * address, that's actually OK. kvm_xen_update_runstate_guest() * will cope. */ - if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode) + if (kvm_xen_has_64bit_shinfo(vcpu->kvm)) sz = sizeof(struct vcpu_runstate_info); else sz = sizeof(struct compat_vcpu_runstate_info); @@ -1439,16 +1461,21 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static inline int max_evtchn_port(struct kvm *kvm) +static inline int max_evtchn_port(bool has_64bit_shinfo) { - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) + if (has_64bit_shinfo) return EVTCHN_2L_NR_CHANNELS; else return COMPAT_EVTCHN_2L_NR_CHANNELS; } -static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, - evtchn_port_t *ports) +static inline int kvm_max_evtchn_port(struct kvm *kvm) +{ + return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm)); +} + +static bool wait_pending_event(struct kvm_vcpu *vcpu, bool has_64bit_shinfo, + int nr_ports, evtchn_port_t *ports) { struct kvm *kvm = vcpu->kvm; struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; @@ -1463,7 +1490,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, goto out_rcu; ret = false; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; } else { @@ -1485,9 +1512,10 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, return ret; } -static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, u64 param, u64 *r) { + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(vcpu->kvm); struct sched_poll sched_poll; evtchn_port_t port, *ports; struct x86_exception e; @@ -1497,7 +1525,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, !(vcpu->kvm->arch.xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND)) return false; - if (IS_ENABLED(CONFIG_64BIT) && !longmode) { + if (IS_ENABLED(CONFIG_64BIT) && !is_64bit) { struct compat_sched_poll sp32; /* Sanity check that the compat struct definition is correct */ @@ -1546,20 +1574,20 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, } for (i = 0; i < sched_poll.nr_ports; i++) { - if (ports[i] >= max_evtchn_port(vcpu->kvm)) { + if (ports[i] >= max_evtchn_port(has_64bit_shinfo)) { *r = -EINVAL; goto out; } } if (sched_poll.nr_ports == 1) - vcpu->arch.xen.poll_evtchn = port; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, port); else - vcpu->arch.xen.poll_evtchn = -1; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, -1); set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); - if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) { + if (!wait_pending_event(vcpu, has_64bit_shinfo, sched_poll.nr_ports, ports)) { kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); if (sched_poll.timeout) @@ -1574,7 +1602,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } - vcpu->arch.xen.poll_evtchn = 0; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, 0); *r = 0; out: /* Really, this is only needed in case of timeout */ @@ -1594,12 +1622,12 @@ static void cancel_evtchn_poll(struct timer_list *t) kvm_vcpu_kick(vcpu); } -static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, u64 param, u64 *r) { switch (cmd) { case SCHEDOP_poll: - if (kvm_xen_schedop_poll(vcpu, longmode, param, r)) + if (kvm_xen_schedop_poll(vcpu, is_64bit, param, r)) return true; fallthrough; case SCHEDOP_yield: @@ -1618,7 +1646,7 @@ struct compat_vcpu_set_singleshot_timer { uint32_t flags; } __attribute__((packed)); -static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, +static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, int vcpu_id, u64 param, u64 *r) { struct vcpu_set_singleshot_timer oneshot; @@ -1662,7 +1690,7 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) != sizeof_field(struct vcpu_set_singleshot_timer, flags)); - if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) : + if (kvm_read_guest_virt(vcpu, param, &oneshot, is_64bit ? sizeof(oneshot) : sizeof(struct compat_vcpu_set_singleshot_timer), &e)) { *r = -EFAULT; return true; @@ -1694,7 +1722,7 @@ static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout, int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { - bool longmode; + bool is_64bit; u64 input, params[6], r = -ENOSYS; bool handled = false; u8 cpl; @@ -1704,8 +1732,8 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) kvm_hv_hypercall_enabled(vcpu)) return kvm_hv_hypercall(vcpu); - longmode = is_64_bit_hypercall(vcpu); - if (!longmode) { + is_64bit = is_64_bit_hypercall(vcpu); + if (!is_64bit) { input = kvm_eax_read(vcpu); params[0] = kvm_ebx_read(vcpu); params[1] = kvm_ecx_read(vcpu); @@ -1751,17 +1779,17 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); break; case __HYPERVISOR_sched_op: - handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0], + handled = kvm_xen_hcall_sched_op(vcpu, is_64bit, params[0], params[1], &r); break; case __HYPERVISOR_vcpu_op: - handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1], + handled = kvm_xen_hcall_vcpu_op(vcpu, is_64bit, params[0], params[1], params[2], &r); break; case __HYPERVISOR_set_timer_op: { u64 timeout = params[0]; /* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */ - if (!longmode) + if (!is_64bit) timeout |= params[1] << 32; handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r); break; @@ -1776,7 +1804,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handle_in_userspace: vcpu->run->exit_reason = KVM_EXIT_XEN; vcpu->run->xen.type = KVM_EXIT_XEN_HCALL; - vcpu->run->xen.u.hcall.longmode = longmode; + vcpu->run->xen.u.hcall.longmode = is_64bit; vcpu->run->xen.u.hcall.cpl = cpl; vcpu->run->xen.u.hcall.input = input; vcpu->run->xen.u.hcall.params[0] = params[0]; @@ -1794,7 +1822,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) { - int poll_evtchn = vcpu->arch.xen.poll_evtchn; + int poll_evtchn = READ_ONCE(vcpu->arch.xen.poll_evtchn); if ((poll_evtchn == port || poll_evtchn == -1) && test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) { @@ -1816,8 +1844,9 @@ static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm); + unsigned long *pending_bits, *mask_bits, vi_pending_sel_ofs; struct kvm_vcpu *vcpu; - unsigned long *pending_bits, *mask_bits; unsigned long flags; int port_word_bit; bool kick_vcpu = false; @@ -1833,7 +1862,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } - if (xe->port >= max_evtchn_port(kvm)) + if (xe->port >= max_evtchn_port(has_64bit_shinfo)) return -EINVAL; rc = -EWOULDBLOCK; @@ -1844,16 +1873,23 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) if (!kvm_gpc_check(gpc, PAGE_SIZE)) goto out_rcu; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 64; + + vi_pending_sel_ofs = offsetof(struct vcpu_info, evtchn_pending_sel); } else { struct compat_shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 32; + + vi_pending_sel_ofs = offsetof(struct compat_vcpu_info, evtchn_pending_sel); + + /* test_and_set_bit() needs 64-bit alignment, but that's OK */ + BUILD_BUG_ON(offsetof(struct compat_shared_info, evtchn_pending) & 7); } /* @@ -1869,6 +1905,8 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) rc = -ENOTCONN; /* Masked */ kvm_xen_check_poller(vcpu, xe->port); } else { + bool old; + rc = 1; /* Delivered to the bitmap in shared_info. */ /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ read_unlock_irqrestore(&gpc->lock, flags); @@ -1885,19 +1923,29 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { - struct vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } - } else { - struct compat_vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, - (unsigned long *)&vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } + /* + * Explicitly use a 32-bit btsl instead of test_and_set_bit(), + * which would use btsq on x86-64. The vcpu_info is guest- + * controlled and only required to be 32-bit aligned, so a + * 64-bit access could generate a split-lock #AC. + * + * Note, this does not apply to the test_and_set_bit() on + * pending_bits above: that is in the per-VM shared_info, which + * is page aligned, so the access is guaranteed to be 64-bit + * aligned. + */ + old = GEN_BINARY_RMWcc(LOCK_PREFIX "btsl", + *(u32 *)(gpc->khva + vi_pending_sel_ofs), + c, "Ir", port_word_bit); + if (!old) { + struct vcpu_info *vi = gpc->khva; + + /* No need for compat handling */ + BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) != + offsetof(struct compat_vcpu_info, evtchn_upcall_pending)); + + WRITE_ONCE(vi->evtchn_upcall_pending, 1); + kick_vcpu = true; } /* For the per-vCPU lapic vector, deliver it as MSI. */ @@ -1995,7 +2043,7 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, struct kvm_vcpu *vcpu; /* - * Don't check for the port being within range of max_evtchn_port(). + * Don't check for the port being within range of kvm_max_evtchn_port(). * Userspace can configure what ever targets it likes; events just won't * be delivered if/while the target is invalid, just like userspace can * configure MSIs which target non-existent APICs. @@ -2004,8 +2052,8 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, * can be restored *independently* of other things like creating vCPUs, * without imposing an ordering dependency on userspace. In this * particular case, the problematic ordering would be with setting the - * Xen 'long mode' flag, which changes max_evtchn_port() to allow 4096 - * instead of 1024 event channels. + * Xen 'long mode' flag, which changes kvm_max_evtchn_port() to allow + * 4096 instead of 1024 event channels. */ /* We only support 2 level event channels for now */ @@ -2042,7 +2090,7 @@ int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn * struct kvm_xen_evtchn e; int ret; - if (!uxe->port || uxe->port >= max_evtchn_port(kvm)) + if (!uxe->port || uxe->port >= kvm_max_evtchn_port(kvm)) return -EINVAL; /* We only support 2 level event channels for now */ @@ -2093,7 +2141,7 @@ static int kvm_xen_eventfd_update(struct kvm *kvm, /* Protect writes to evtchnfd as well as the idr lookup. */ mutex_lock(&kvm->arch.xen.xen_lock); - evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port); + evtchnfd = xa_load(&kvm->arch.xen.evtchn_ports, port); ret = -ENOENT; if (!evtchnfd) @@ -2152,7 +2200,7 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, case EVTCHNSTAT_interdomain: if (data->u.evtchn.deliver.port.port) { - if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm)) + if (data->u.evtchn.deliver.port.port >= kvm_max_evtchn_port(kvm)) goto out_noeventfd; /* -EINVAL */ } else { eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd); @@ -2187,13 +2235,13 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, } mutex_lock(&kvm->arch.xen.xen_lock); - ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1, + ret = xa_insert(&kvm->arch.xen.evtchn_ports, port, evtchnfd, GFP_KERNEL); mutex_unlock(&kvm->arch.xen.xen_lock); - if (ret >= 0) + if (!ret) return 0; - if (ret == -ENOSPC) + if (ret == -EBUSY) ret = -EEXIST; out: if (eventfd) @@ -2208,7 +2256,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) struct evtchnfd *evtchnfd; mutex_lock(&kvm->arch.xen.xen_lock); - evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port); + evtchnfd = xa_erase(&kvm->arch.xen.evtchn_ports, port); mutex_unlock(&kvm->arch.xen.xen_lock); if (!evtchnfd) @@ -2224,7 +2272,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) static int kvm_xen_eventfd_reset(struct kvm *kvm) { struct evtchnfd *evtchnfd, **all_evtchnfds; - int i; + unsigned long i; int n = 0; mutex_lock(&kvm->arch.xen.xen_lock); @@ -2234,7 +2282,7 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm) * critical section, first collect all the evtchnfd objects * in an array as they are removed from evtchn_ports. */ - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) n++; all_evtchnfds = kmalloc_objs(struct evtchnfd *, n); @@ -2244,9 +2292,9 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm) } n = 0; - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) { all_evtchnfds[n++] = evtchnfd; - idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); + xa_erase(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); } mutex_unlock(&kvm->arch.xen.xen_lock); @@ -2270,7 +2318,7 @@ static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET) return kvm_xen_eventfd_reset(kvm); - if (!port || port >= max_evtchn_port(kvm)) + if (!port || port >= kvm_max_evtchn_port(kvm)) return -EINVAL; if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN) @@ -2297,12 +2345,10 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) } /* - * evtchnfd is protected by kvm->srcu; the idr lookup instead - * is protected by RCU. + * evtchnfd is protected by kvm->srcu; the xa_load is RCU-safe + * internally, no explicit rcu_read_lock() needed. */ - rcu_read_lock(); - evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port); - rcu_read_unlock(); + evtchnfd = xa_load(&vcpu->kvm->arch.xen.evtchn_ports, send.port); if (!evtchnfd) return false; @@ -2349,23 +2395,23 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) void kvm_xen_init_vm(struct kvm *kvm) { mutex_init(&kvm->arch.xen.xen_lock); - idr_init(&kvm->arch.xen.evtchn_ports); + xa_init(&kvm->arch.xen.evtchn_ports); kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm); } void kvm_xen_destroy_vm(struct kvm *kvm) { struct evtchnfd *evtchnfd; - int i; + unsigned long i; kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache); - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) { if (!evtchnfd->deliver.port.port) eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx); kfree(evtchnfd); } - idr_destroy(&kvm->arch.xen.evtchn_ports); + xa_destroy(&kvm->arch.xen.evtchn_ports); if (kvm->arch.xen.hvm_config.msr) static_branch_slow_dec_deferred(&kvm_xen_enabled);
diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index f372855..9d04e35 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h
@@ -235,6 +235,11 @@ struct compat_shared_info { #define COMPAT_EVTCHN_2L_NR_CHANNELS (8 * \ sizeof_field(struct compat_shared_info, \ evtchn_pending)) + +/* Latched VM-wide mode; the KVM equivalent of Xen's !has_32bit_shinfo(). */ +#define kvm_xen_has_64bit_shinfo(kvm) \ + (IS_ENABLED(CONFIG_64BIT) && READ_ONCE((kvm)->arch.xen.long_mode)) + struct compat_vcpu_runstate_info { int state; uint64_t state_entry_time;
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 6bca874..6a2fe3d 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c
@@ -9,11 +9,13 @@ #include <linux/filter.h> #include <linux/if_vlan.h> #include <linux/bitfield.h> +#include <linux/init.h> #include <linux/bpf.h> #include <linux/bpf_verifier.h> #include <linux/memory.h> #include <linux/sort.h> #include <linux/execmem.h> +#include <linux/extable.h> #include <asm/extable.h> #include <asm/ftrace.h> #include <asm/set_memory.h> @@ -22,8 +24,34 @@ #include <asm/unwind.h> #include <asm/cfi.h> +#if IS_ENABLED(CONFIG_BPF_JIT_KASAN) +void __asan_load1(void *p); +void __asan_store1(void *p); +void __asan_load2(void *p); +void __asan_store2(void *p); +void __asan_load4(void *p); +void __asan_store4(void *p); +void __asan_load8(void *p); +void __asan_store8(void *p); +#endif + static bool all_callee_regs_used[4] = {true, true, true, true}; +static void *trampoline_size_image; + +static int __init init_trampoline_size_image(void) +{ + /* + * The generated trampoline contains calls and jumps with 32bit relative + * offsets, so the scratch image must be in the execmem range. + * On x86, module data and executable memory share the same address range, + * so using EXECMEM_MODULE_DATA to get writable memory. + */ + trampoline_size_image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); + return trampoline_size_image ? 0 : -ENOMEM; +} +late_initcall(init_trampoline_size_image); + static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) { if (len == 1) @@ -209,8 +237,16 @@ static const int reg2pt_regs[] = { [BPF_REG_7] = offsetof(struct pt_regs, r13), [BPF_REG_8] = offsetof(struct pt_regs, r14), [BPF_REG_9] = offsetof(struct pt_regs, r15), + [BPF_REG_AX] = offsetof(struct pt_regs, r10), }; +static bool is_atomic_fetch_op(const struct bpf_insn *insn) +{ + return insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH); +} + /* * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 * which need extra byte of encoding. @@ -393,7 +429,7 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) } /* add rsp, depth */ -static void emit_add_rsp(u8 **pprog, u16 depth) +static void emit_add_rsp(u8 **pprog, u32 depth) { u8 *prog = *pprog; @@ -407,7 +443,7 @@ static void emit_add_rsp(u8 **pprog, u16 depth) } /* sub rsp, depth */ -static void emit_sub_rsp(u8 **pprog, u16 depth) +static void emit_sub_rsp(u8 **pprog, u32 depth) { u8 *prog = *pprog; @@ -444,6 +480,9 @@ static void emit_nops(u8 **pprog, int len) * in arch/x86/kernel/alternative.c */ static int emit_call(u8 **prog, void *func, void *ip); +static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off); +static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg); +static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64); static void emit_fineibt(u8 **pprog, u8 *ip, u32 hash, int arity) { @@ -579,8 +618,7 @@ static void emit_prologue(u8 **pprog, u8 *ip, u32 stack_depth, bool ebpf_from_cb EMIT_ENDBR(); /* sub rsp, rounded_stack_depth */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); + emit_sub_rsp(&prog, round_up(stack_depth, 8)); if (tail_call_reachable) emit_prologue_tail_call(&prog, is_subprog); *pprog = prog; @@ -722,6 +760,46 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip) *pprog = prog; } +static void __emit_indirect_call(u8 **pprog, int reg, bool ereg) +{ + u8 *prog = *pprog; + + if (ereg) + EMIT1(0x41); + + EMIT2(0xFF, 0xD0 + reg); + + *pprog = prog; +} + +/* call *bpf_reg */ +static int emit_indirect_call(u8 **pprog, int bpf_reg, u8 *ip) +{ + u8 *prog = *pprog; + int reg = reg2hex[bpf_reg]; + bool ereg = is_ereg(bpf_reg); + int err = 0; + + if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) { + OPTIMIZER_HIDE_VAR(reg); + err = emit_call(&prog, its_static_thunk(reg + 8*ereg), ip); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + EMIT_LFENCE(); + __emit_indirect_call(&prog, reg, ereg); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { + OPTIMIZER_HIDE_VAR(reg); + if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) + err = emit_call(&prog, &__x86_indirect_call_thunk_array[reg + 8*ereg], ip); + else + err = emit_call(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip); + } else { + __emit_indirect_call(&prog, reg, ereg); + } + + *pprog = prog; + return err; +} + static void emit_return(u8 **pprog, u8 *ip) { u8 *prog = *pprog; @@ -772,7 +850,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * if (index >= array->map.max_entries) * goto out; */ - EMIT2(0x89, 0xD2); /* mov edx, edx */ + emit_mov_reg(&prog, false, BPF_REG_3, BPF_REG_3); /* mov edx, edx */ EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ offsetof(struct bpf_array, map.max_entries)); @@ -783,7 +861,8 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) * goto out; */ - EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, tcc_ptr_off); EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_indirect_label - (prog + 2 - start); @@ -810,7 +889,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -820,15 +899,13 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * Pop tail_call_cnt_ptr, if it's subprog. */ EMIT1(0x58); /* pop rax */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ - round_up(stack_depth, 8)); + emit_add_rsp(&prog, round_up(stack_depth, 8)); /* add rsp, sd */ /* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */ EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ offsetof(struct bpf_prog, bpf_func)); - EMIT4(0x48, 0x83, 0xC1, /* add rcx, X86_TAIL_CALL_OFFSET */ - X86_TAIL_CALL_OFFSET); + /* add rcx, X86_TAIL_CALL_OFFSET */ + emit_alu_imm(&prog, BPF_ADD, BPF_REG_4, X86_TAIL_CALL_OFFSET, true); /* * Now we're ready to jump into next BPF program * rdi == ctx (1st arg) @@ -855,7 +932,8 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) * goto out; */ - EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, tcc_ptr_off); EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_direct_label - (prog + 2 - start); @@ -877,7 +955,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -887,8 +965,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, * Pop tail_call_cnt_ptr, if it's subprog. */ EMIT1(0x58); /* pop rax */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); + emit_add_rsp(&prog, round_up(stack_depth, 8)); emit_nops(&prog, X86_PATCH_SIZE); @@ -1038,8 +1115,9 @@ static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, } else { /* movs[b,w]l dst, src */ if (num_bits == 8) { - EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe, - add_2reg(0xC0, src_reg, dst_reg)); + if (is_ereg(dst_reg) || is_ereg_8l(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT3(0x0f, 0xbe, add_2reg(0xC0, src_reg, dst_reg)); } else if (num_bits == 16) { if (is_ereg(dst_reg) || is_ereg(src_reg)) EMIT1(add_2mod(0x40, src_reg, dst_reg)); @@ -1051,18 +1129,40 @@ static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, *pprog = prog; } +/* + * rbp and r13 as base register always need a displacement, since mod 00 + * with r/m 101 encodes a RIP-relative disp32 and with SIB base 101 a + * disp32 without base instead. + */ +static bool base_needs_disp(u32 reg) +{ + return reg2hex[reg] == 5; +} + +static void emit_movzx_reg(u8 **pprog, int num_bits, u32 dst_reg, u32 src_reg) +{ + u8 *prog = *pprog; + + /* movz[b,w]l dst, src */ + if (is_ereg(dst_reg) || + (num_bits == 8 ? is_ereg_8l(src_reg) : is_ereg(src_reg))) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT3(0x0F, num_bits == 8 ? 0xB6 : 0xB7, + add_2reg(0xC0, src_reg, dst_reg)); + + *pprog = prog; +} + /* Emit the suffix (ModR/M etc) for addressing *(ptr_reg + off) and val_reg */ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) { u8 *prog = *pprog; - if (is_imm8(off)) { - /* 1-byte signed displacement. - * - * If off == 0 we could skip this and save one extra byte, but - * special case of x86 R13 which always needs an offset is not - * worth the hassle - */ + if (!off && !base_needs_disp(ptr_reg)) { + /* No displacement */ + EMIT1(add_2reg(0x00, ptr_reg, val_reg)); + } else if (is_imm8(off)) { + /* 1-byte signed displacement */ EMIT2(add_2reg(0x40, ptr_reg, val_reg), off); } else { /* 4-byte signed displacement */ @@ -1071,14 +1171,22 @@ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) *pprog = prog; } -static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off) +/* + * Emit the suffix for addressing *(ptr_reg + (index_reg << scale) + off) + * and val_reg, with a SIB byte + */ +static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off, + u8 scale) { + u8 sib = (scale << 6) | add_2reg(0, ptr_reg, index_reg); u8 *prog = *pprog; - if (is_imm8(off)) { - EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + if (!off && !base_needs_disp(ptr_reg)) { + EMIT2(add_2reg(0x04, BPF_REG_0, val_reg), sib); + } else if (is_imm8(off)) { + EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), sib, off); } else { - EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), sib, off); } *pprog = prog; } @@ -1111,6 +1219,165 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +/* add/sub/and/or/xor dst_reg, imm with the imm8 and the rax short forms */ +static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64) +{ + /* ModR/M opcode extension and rax short form per operation */ + static const u8 modrm[] = { + [BPF_ADD] = 0xC0, [BPF_SUB] = 0xE8, [BPF_AND] = 0xE0, + [BPF_OR] = 0xC8, [BPF_XOR] = 0xF0, + }; + static const u8 short_op[] = { + [BPF_ADD] = 0x05, [BPF_SUB] = 0x2D, [BPF_AND] = 0x25, + [BPF_OR] = 0x0D, [BPF_XOR] = 0x35, + }; + u8 *prog = *pprog; + bool inc; + + /* A non-negative mask clears the upper half like the 32-bit and does */ + if (op == BPF_AND && imm >= 0) + is64 = false; + /* add 128 is sub -128 and the other way round, which fit into an imm8 */ + if ((op == BPF_ADD || op == BPF_SUB) && imm == 128) { + op = op == BPF_ADD ? BPF_SUB : BPF_ADD; + imm = -128; + } + inc = op == BPF_ADD ? imm == 1 : imm == -1; + maybe_emit_1mod(&prog, dst_reg, is64); + /* inc or dec for +-1, the flags are not read across instructions */ + if ((op == BPF_ADD || op == BPF_SUB) && (imm == 1 || imm == -1)) + EMIT2(0xFF, add_1reg(inc ? 0xC0 : 0xC8, dst_reg)); + else if (is_imm8(imm)) + EMIT3(0x83, add_1reg(modrm[op], dst_reg), imm); + else if (is_axreg(dst_reg)) + EMIT1_off32(short_op[op], imm); + else + EMIT2_off32(0x81, add_1reg(modrm[op], dst_reg), imm); + + *pprog = prog; +} + +/* add/sub/and/or/xor dst_reg, src_reg */ +static void emit_alu_reg(u8 **pprog, u32 op, u32 dst_reg, u32 src_reg, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_mod(&prog, dst_reg, src_reg, is64); + EMIT2(simple_alu_opcodes[op], add_2reg(0xC0, dst_reg, src_reg)); + + *pprog = prog; +} + +/* shl/shr/sar reg, imm */ +static void emit_shift_imm(u8 **pprog, u32 op, u32 reg, u8 imm, bool is64) +{ + u8 *prog = *pprog; + + /* The shift left by one is an add of the register to itself */ + if (op == BPF_LSH && imm == 1) { + emit_alu_reg(&prog, BPF_ADD, reg, reg, is64); + } else { + maybe_emit_1mod(&prog, reg, is64); + if (imm == 1) + EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); + else + EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + } + + *pprog = prog; +} + +/* neg dst_reg */ +static void emit_neg(u8 **pprog, u32 dst_reg, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_1mod(&prog, dst_reg, is64); + EMIT2(0xF7, add_1reg(0xD8, dst_reg)); + + *pprog = prog; +} + +static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, + u32 addr_reg, s32 off, u32 bpf_size, u8 *ip, + bool is_write) +{ +#ifdef CONFIG_BPF_JIT_KASAN + u8 *prog = *pprog; + void *kasan_func; + + if (!env) + return 0; + + if (addr_reg == BPF_REG_FP || addr_reg == BPF_REG_PARAMS) + return 0; + + /* Derive KASAN check function from access type and size */ + switch (bpf_size) { + case BPF_B: + kasan_func = is_write ? __asan_store1 : __asan_load1; + break; + case BPF_H: + kasan_func = is_write ? __asan_store2 : __asan_load2; + break; + case BPF_W: + kasan_func = is_write ? __asan_store4 : __asan_load4; + break; + case BPF_DW: + kasan_func = is_write ? __asan_store8 : __asan_load8; + break; + default: + return -EINVAL; + } + + /* Save rax */ + EMIT1(0x50); + /* Save rcx */ + EMIT1(0x51); + /* Save rdx */ + EMIT1(0x52); + /* Save rsi */ + EMIT1(0x56); + /* Save rdi */ + EMIT1(0x57); + /* Save r8 */ + EMIT2(0x41, 0x50); + /* Save r9 */ + EMIT2(0x41, 0x51); + /* Save r10 */ + EMIT2(0x41, 0x52); + /* Save r11 */ + EMIT2(0x41, 0x53); + /* mov rdi, addr_reg */ + EMIT_mov(BPF_REG_1, addr_reg); + + /* add rdi, off (if offset is non-zero) */ + if (off) + emit_alu_imm(&prog, BPF_ADD, BPF_REG_1, off, true); + + /* Adjust ip to account for the instrumentation generated so far */ + ip += (prog - *pprog); + /* We emit a call, so update call depth counting */ + ip += x86_call_depth_emit_accounting(&prog, kasan_func, ip); + /* call kasan_func */ + if (emit_call(&prog, kasan_func, ip)) + return -ERANGE; + + EMIT2(0x41, 0x5B); + EMIT2(0x41, 0x5A); + EMIT2(0x41, 0x59); + EMIT2(0x41, 0x58); + EMIT1(0x5F); + EMIT1(0x5E); + EMIT1(0x5A); + EMIT1(0x59); + EMIT1(0x58); + + *pprog = prog; +#endif /* CONFIG_BPF_JIT_KASAN */ + return 0; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -1118,12 +1385,16 @@ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) switch (size) { case BPF_B: - /* Emit 'movzx rax, byte ptr [rax + off]' */ - EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); + /* Emit 'movzx eax, byte ptr [rax + off]' */ + if (is_ereg(dst_reg) || is_ereg(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT2(0x0F, 0xB6); break; case BPF_H: - /* Emit 'movzx rax, word ptr [rax + off]' */ - EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); + /* Emit 'movzx eax, word ptr [rax + off]' */ + if (is_ereg(dst_reg) || is_ereg(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT2(0x0F, 0xB7); break; case BPF_W: /* Emit 'mov eax, dword ptr [rax+0x14]' */ @@ -1186,7 +1457,7 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x8B); break; } - emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off, 0); *pprog = prog; } @@ -1208,7 +1479,7 @@ static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x63); break; } - emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off, 0); *pprog = prog; } @@ -1279,7 +1550,7 @@ static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i EMIT2(add_3mod(0x48, dst_reg, src_reg, index_reg), 0x89); break; } - emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off, 0); *pprog = prog; } @@ -1311,11 +1582,47 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int EMIT2(add_3mod(0x48, dst_reg, 0, index_reg), 0xC7); break; } - emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off, 0); EMIT(imm, bpf_size_to_x86_bytes(size)); *pprog = prog; } +/* ST: *(u8*)(dst_reg + off) = imm */ +static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + s32 insn_off) +{ + s32 imm32 = insn->imm; + u8 *prog = *pprog; + + switch (BPF_SIZE(insn->code)) { + case BPF_B: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC6); + else + EMIT1(0xC6); + break; + case BPF_H: + if (is_ereg(dst_reg)) + EMIT3(0x66, 0x41, 0xC7); + else + EMIT2(0x66, 0xC7); + break; + case BPF_W: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC7); + else + EMIT1(0xC7); + break; + case BPF_DW: + EMIT2(add_1mod(0x48, dst_reg), 0xC7); + break; + } + + emit_insn_suffix(&prog, dst_reg, BPF_REG_0, insn_off); + EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + *pprog = prog; +} + static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) { emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); @@ -1419,22 +1726,40 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op); return -EFAULT; } - emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off, 0); *pprog = prog; return 0; } -static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg, - u32 src_reg, s16 off, u8 bpf_size) +static int emit_atomic_ld_st(struct bpf_verifier_env *env, u8 **pprog, + struct bpf_insn *insn, u8 *ip, u32 dst_reg, + u32 src_reg, bool accesses_stack_only) { + u32 atomic_op = insn->imm; + int err; + switch (atomic_op) { case BPF_LOAD_ACQ: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, src_reg, insn->off, + BPF_SIZE(insn->code), ip, false); + if (err) + return err; + } /* dst_reg = smp_load_acquire(src_reg + off16) */ - emit_ldx(pprog, bpf_size, dst_reg, src_reg, off); + emit_ldx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; case BPF_STORE_REL: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, dst_reg, insn->off, + BPF_SIZE(insn->code), ip, true); + if (err) + return err; + } /* smp_store_release(dst_reg + off16, src_reg) */ - emit_stx(pprog, bpf_size, dst_reg, src_reg, off); + emit_stx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; default: pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", @@ -1468,6 +1793,93 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, } /* + * A fetching AND/OR/XOR can't be implemented with a single x86 insn, so do a + * CMPXCHG loop. @index_reg is X86_REG_R12 for an arena access or -1 otherwise. + * Unless @r0_on_stack, the caller has substituted BPF_REG_AX for R0 in + * @dst_reg and @src_reg. + * + * Both the load and the CMPXCHG can fault when accessing the arena, so their + * addresses are handed back in @fault[0] and @fault[1]. A fault has to resume + * at @resume, which is past the loop and past the move that delivers the old + * value, but before R0 is restored. Both are NULL for the non-arena case, + * which needs no fixup. + * + * R0 is expected to be saved in BPF_REG_AX by the caller, or pushed on the + * stack if @r0_on_stack. In that case an R0 operand stays in the stack slot, + * RAX ends up holding the old value that R0 is to return, and the slot is + * dropped rather than popped. + */ +static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + u32 src_reg, int index_reg, bool r0_on_stack, + u8 **fault, u8 **resume) +{ + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + bool operand_on_stack = r0_on_stack && src_reg == BPF_REG_0; + u8 *branch_target, *prog = *pprog; + int err; + + branch_target = prog; + + /* Load old value */ + if (fault) + fault[0] = prog; + if (index_reg < 0) + emit_ldx(&prog, BPF_SIZE(insn->code), BPF_REG_0, dst_reg, insn->off); + else + emit_ldx_index(&prog, BPF_SIZE(insn->code), BPF_REG_0, dst_reg, + index_reg, insn->off); + + /* + * Perform the (commutative) operation locally, put the result in + * the AUX_REG. + */ + if (operand_on_stack) { + /* mov r11, [rsp]: the operand is the R0 saved there, RAX is the old value */ + if (is64) + EMIT4(0x4C, 0x8B, 0x1C, 0x24); + else + EMIT4(0x44, 0x8B, 0x1C, 0x24); + } else { + emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); + } + maybe_emit_mod(&prog, AUX_REG, src_reg, is64); + EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], + add_2reg(0xC0, AUX_REG, src_reg)); + + /* Attempt to swap in new value */ + if (fault) + fault[1] = prog; + if (index_reg < 0) + err = emit_atomic_rmw(&prog, BPF_CMPXCHG, dst_reg, AUX_REG, + insn->off, BPF_SIZE(insn->code)); + else + err = emit_atomic_rmw_index(&prog, BPF_CMPXCHG, BPF_SIZE(insn->code), + dst_reg, AUX_REG, index_reg, insn->off); + if (WARN_ON(err)) + return err; + + /* ZF tells us whether we won the race. If it's cleared we need to try again. */ + EMIT2(X86_JNE, -(prog - branch_target) - 2); + /* Return the pre-modification value, which RAX already is for R0 */ + if (!operand_on_stack) + emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); + + if (resume) + *resume = prog; + + /* Restore R0 after clobbering RAX */ + if (!r0_on_stack) + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); + else if (operand_on_stack) + EMIT2(0x41, 0x5B); /* pop r11, dropping the operand */ + else + EMIT1(0x58); /* pop rax */ + + *pprog = prog; + return 0; +} + +/* * Metadata encoding for exception handling in JITed code. * * Format of `fixup` and `data` fields in `struct exception_table_entry`: @@ -1480,7 +1892,12 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, * | ARENA_ACC | ARENA_WRITE | Unused | ARENA_REG | DST_REG | INSN_LEN | * +-----------+-------------+--------+-----------+---------+----------+ * - * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). + * - INSN_LEN (8 bits): How far past the faulting insn to resume. That is its own length + * for a single-insn access, but the distance to the end of the whole + * sequence where one BPF insn became several, as for the CMPXCHG loop + * of a fetching AND/OR/XOR. A sequence long enough to overflow this + * field would first have to exceed BPF_MAX_INSN_SIZE, which do_jit() + * rejects with -EFAULT before the image is used. * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). * This is set to DONT_CLEAR if the insn does not read into a register. * - ARENA_REG (8 bits): Offset of the register that is used to calculate the @@ -1527,7 +1944,7 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) bpf_prog_report_arena_violation(is_write, addr, regs->ip); } - /* jump over faulting load and clear dest register */ + /* jump over the faulting access, or the sequence it is part of, and clear dest register */ if (reg != DONT_CLEAR) *(unsigned long *)((void *)regs + reg) = 0; regs->ip += insn_len; @@ -1535,6 +1952,44 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) return true; } +/* + * Record an arena access that may fault. @fault_ip is the address of the + * faulting insn in the RO image, @resume_off how far past it execution has to + * resume: for a multi-insn lowering that is the end of the whole sequence, not + * the end of the one insn. + */ +static int emit_arena_exentry(struct bpf_prog *bpf_prog, u8 *image, u8 *rw_image, + int *excnt, u8 *fault_ip, u32 resume_off, + u32 fixup_reg, u32 arena_reg, bool is_write, s16 off) +{ + struct exception_table_entry *ex; + s64 delta; + + if (!bpf_prog->aux->extable) + return 0; + + if (*excnt >= bpf_prog->aux->num_exentries) { + pr_err("arena extable bug\n"); + return -EFAULT; + } + ex = &bpf_prog->aux->extable[(*excnt)++]; + + delta = fault_ip - (u8 *)&ex->insn; + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + + ex->insn = delta; + ex->data = EX_TYPE_BPF | FIELD_PREP(DATA_ARENA_OFFSET_MASK, off); + ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, resume_off) | + FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | + FIELD_PREP(FIXUP_REG_MASK, fixup_reg) | + FIXUP_ARENA_ACCESS; + if (is_write) + ex->fixup |= FIXUP_ARENA_WRITE; + + return 0; +} + static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt, bool *regs_used) { @@ -1611,6 +2066,218 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) *pprog = prog; } +struct bpf_div_magic { + u64 m; /* multiplier, sign extended for signed division */ + int s; /* shift of the high half of the product */ + bool add; /* multiplier has 65 bits, use the add sequence */ +}; + +/* + * Multiplier and shift for an unsigned division by the constant d, with + * 2 <= d < 2^64 and a dividend below 2^(64 - lz): n / d == (mulhu(n, m) + * >> s), or with the add sequence when m needs 65 bits, (((n - t) >> 1) + * + t) >> (s - 1) with t = mulhu(n, m). Hacker's Delight, 2nd ed., figure + * 10-2. + */ +static void bpf_magic_udiv(u64 d, int lz, struct bpf_div_magic *mag) +{ + u64 mask = ~0ULL >> (lz & 63); + u64 nc, delta, q1, r1, q2, r2; + int p; + + mag->add = false; + /* Largest value with nc mod d == d - 1 */ + nc = mask - (mask % d + 1) % d; + p = 63; + q1 = (1ULL << 63) / nc; /* 2^p / nc */ + r1 = (1ULL << 63) - q1 * nc; /* 2^p mod nc */ + q2 = (~0ULL >> 1) / d; /* (2^p - 1) / d */ + r2 = (~0ULL >> 1) - q2 * d; /* (2^p - 1) mod d */ + do { + p++; + if (r1 >= nc - r1) { + q1 = 2 * q1 + 1; + r1 = 2 * r1 - nc; + } else { + q1 = 2 * q1; + r1 = 2 * r1; + } + if (r2 + 1 >= d - r2) { + if (q2 >= (~0ULL >> 1)) + mag->add = true; + q2 = 2 * q2 + 1; + r2 = 2 * r2 + 1 - d; + } else { + if (q2 >= (1ULL << 63)) + mag->add = true; + q2 = 2 * q2; + r2 = 2 * r2 + 1; + } + delta = d - 1 - r2; + } while (p < 128 && (q1 < delta || (q1 == delta && r1 == 0))); + + mag->m = q2 + 1; + mag->s = p - 64; +} + +/* + * Multiplier and shift for a signed division by the constant d, with + * 2 <= |d| < 2^63: q = mulhs(n, m), q += n if d > 0 and m < 0, q -= n + * if d < 0 and m > 0, q >>= s (arithmetic), n / d == q + (q >>> 63). + * Hacker's Delight, 2nd ed., figure 10-1. + */ +static void bpf_magic_sdiv(s64 d, struct bpf_div_magic *mag) +{ + u64 ad = d < 0 ? -(u64)d : (u64)d; + u64 t, anc, delta, q1, r1, q2, r2; + int p; + + t = (1ULL << 63) + (d < 0); + anc = t - 1 - t % ad; /* |nc| */ + p = 63; + q1 = (1ULL << 63) / anc; /* 2^p / |nc| */ + r1 = (1ULL << 63) - q1 * anc; /* 2^p mod |nc| */ + q2 = (1ULL << 63) / ad; /* 2^p / |d| */ + r2 = (1ULL << 63) - q2 * ad; /* 2^p mod |d| */ + do { + p++; + q1 = 2 * q1; + r1 = 2 * r1; + if (r1 >= anc) { + q1++; + r1 -= anc; + } + q2 = 2 * q2; + r2 = 2 * r2; + if (r2 >= ad) { + q2++; + r2 -= ad; + } + delta = ad - r2; + } while (q1 < delta || (q1 == delta && r1 == 0)); + + mag->m = q2 + 1; + if (d < 0) + mag->m = -mag->m; + mag->s = p - 64; + mag->add = false; +} + +/* + * Divide dst_reg by the constant in insn->imm of a 64-bit division through + * a multiplication with the magic number from bpf_magic_udiv() or + * bpf_magic_sdiv(). The dividend goes through rax and the multiplier + * through r11, the caller saves rax and rdx unless they are dst_reg, and + * r10 is scratch. Returns false for the divisors left to div/idiv. + */ +static bool emit_div_const(u8 **pprog, const struct bpf_insn *insn, u32 dst_reg) +{ + bool is_mod = BPF_OP(insn->code) == BPF_MOD; + bool is_signed = insn->off == 1; + u32 n_reg = dst_reg, q_reg; + s32 imm32 = insn->imm; + struct bpf_div_magic mag; + u8 *prog = *pprog; + int corr = 0, pre = 0, s; + + if (is_signed) { + /* |d| >= 2, the verifier already rewrote d == -1 */ + if (imm32 >= -1 && imm32 <= 1) + return false; + bpf_magic_sdiv(imm32, &mag); + /* q += n for d > 0 and m < 0, q -= n for d < 0 and m > 0 */ + if (imm32 > 0 && (s64)mag.m < 0) + corr = 1; + else if (imm32 < 0 && (s64)mag.m > 0) + corr = -1; + } else { + /* d >= 2, a negative imm32 is a divisor above 2^63 */ + if (imm32 < 2) + return false; + bpf_magic_udiv(imm32, 0, &mag); + /* + * Shift the dividend by the trailing zeros of an even divisor + * first, the magic number of the odd part then fits without + * the add sequence. + */ + if (mag.add && !(imm32 & 1)) { + pre = __ffs(imm32) & 31; + bpf_magic_udiv(imm32 >> pre, pre, &mag); + if (mag.add) + return false; + } + } + s = mag.s; + /* + * The shift is at most 63, or 64 with the add sequence which shifts + * by s - 1 and needs s >= 1. Never hit for d >= 2, kept as a guard + * against an unencodable count. + */ + if (s > 64 - !mag.add || (mag.add && !s)) + return false; + + /* Keep the dividend when the multiplication clobbers it */ + if ((is_mod || mag.add || corr) && + (dst_reg == BPF_REG_0 || dst_reg == BPF_REG_3)) { + emit_mov_reg(&prog, true, BPF_REG_AX, dst_reg); + n_reg = BPF_REG_AX; + } + + /* mov rax, dst_reg; shr rax, pre; mov r11, m; mul r11 or imul r11 */ + if (dst_reg != BPF_REG_0) + emit_mov_reg(&prog, true, BPF_REG_0, dst_reg); + if (pre) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, pre, true); + emit_mov_imm64(&prog, AUX_REG, mag.m >> 32, (u32)mag.m); + EMIT3(0x49, 0xF7, is_signed ? 0xEB : 0xE3); + if (is_signed) { + /* add rdx, n_reg or sub rdx, n_reg */ + if (corr) + emit_alu_reg(&prog, corr > 0 ? BPF_ADD : BPF_SUB, + BPF_REG_3, n_reg, true); + /* sar rdx, s */ + if (s) + emit_shift_imm(&prog, BPF_ARSH, BPF_REG_3, s, true); + /* mov rax, rdx; shr rax, 63; add rdx, rax */ + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_3); + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, 63, true); + emit_alu_reg(&prog, BPF_ADD, BPF_REG_3, BPF_REG_0, true); + q_reg = BPF_REG_3; + } else if (!mag.add) { + /* shr rdx, s */ + if (s) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_3, s, true); + q_reg = BPF_REG_3; + } else { + /* mov rax, n_reg; sub rax, rdx; shr rax, 1 */ + emit_mov_reg(&prog, true, BPF_REG_0, n_reg); + emit_alu_reg(&prog, BPF_SUB, BPF_REG_0, BPF_REG_3, true); + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, 1, true); + /* add rax, rdx; shr rax, s - 1 */ + emit_alu_reg(&prog, BPF_ADD, BPF_REG_0, BPF_REG_3, true); + if (s > 1) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, s - 1, true); + q_reg = BPF_REG_0; + } + + if (is_mod) { + /* imul q_reg, q_reg, imm32 */ + maybe_emit_mod(&prog, q_reg, q_reg, true); + if (is_imm8(imm32)) + EMIT3(0x6B, add_2reg(0xC0, q_reg, q_reg), imm32); + else + EMIT2_off32(0x69, add_2reg(0xC0, q_reg, q_reg), imm32); + /* sub n_reg, q_reg */ + emit_alu_reg(&prog, BPF_SUB, n_reg, q_reg, true); + q_reg = n_reg; + } + if (q_reg != dst_reg) + emit_mov_reg(&prog, true, dst_reg, q_reg); + + *pprog = prog; + return true; +} + static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) { u8 *prog = *pprog; @@ -1632,7 +2299,7 @@ static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) #define __LOAD_TCC_PTR(off) \ - EMIT3_off32(0x48, 0x8B, 0x85, off) + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, off) /* mov rax, qword ptr [rbp - rounded_stack_depth - 16] */ #define LOAD_TAIL_CALL_CNT_PTR(stack) \ __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack)) @@ -1683,6 +2350,75 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, return 0; } +static const struct bpf_jit_arg_abi x86_arg_abi = { + .nr_arg_regs = 6, + .backfill_after_stack = true, + .even_stack_align = true, +}; + +static const u8 x86_arg_reg[] = { + BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9, +}; + +static bool x86_arg_on_stack(u8 pos) +{ + return pos >= x86_arg_abi.nr_arg_regs; +} + +static s32 x86_arg_stack_off(u8 pos) +{ + return (pos - x86_arg_abi.nr_arg_regs) * 8; +} + +/* + * Move the arguments the x86-64 ABI places somewhere other than the argument + * slot the BPF calling convention gave them. @stack_base addresses the + * outgoing stack argument area from RBP. Return the number of emitted bytes. + */ +static int emit_kfunc_arg_moves(const struct btf_func_model *fm, s32 stack_base, u8 **pprog) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 nreg = x86_arg_abi.nr_arg_regs; + u8 *prog = *pprog, *start = prog; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&x86_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + bool dst_mem = dst != BPF_JIT_ARG_TMP && dst >= nreg; + bool src_mem = src != BPF_JIT_ARG_TMP && src >= nreg; + + /* + * Take the value into a register: the one it belongs in, the + * scratch when it is carried past its own destination, and + * BPF_REG_AX only to pass one stack slot to another. + */ + if (src == BPF_JIT_ARG_TMP) { + reg = AUX_REG; + } else if (src_mem) { + reg = dst == BPF_JIT_ARG_TMP ? AUX_REG : + dst_mem ? BPF_REG_AX : x86_arg_reg[dst]; + emit_ldx(&prog, BPF_DW, reg, BPF_REG_FP, + stack_base + (src - nreg) * 8); + } else { + reg = x86_arg_reg[src]; + } + + /* And leave it where the argument belongs. */ + if (dst == BPF_JIT_ARG_TMP) + emit_mov_reg(&prog, true, AUX_REG, reg); + else if (dst_mem) + emit_stx(&prog, BPF_DW, BPF_REG_FP, reg, + stack_base + (dst - nreg) * 8); + else if (reg != x86_arg_reg[dst]) + emit_mov_reg(&prog, true, x86_arg_reg[dst], reg); + } + + *pprog = prog; + return prog - start; +} + /* * Rebase the __arena args of a kfunc call to arena kernel addresses, * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable @@ -1690,24 +2426,25 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; - int i; + int i, slot; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; - u32 reg = BPF_REG_1 + i; + u32 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = BPF_REG_1 + slot; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; - if (WARN_ON_ONCE(!bpf_prog->aux->arena)) + if (WARN_ON_ONCE(!bpf_prog_uses_arena_insns(bpf_prog))) return -EINVAL; /* mov eN, eN: truncate and clear the upper 32 bits */ @@ -1727,6 +2464,116 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, return prog - start; } +/* + * A strict compare against 1 or 128 is the non-strict one against 0 or + * 127, a strict one against -129 or, with a signed condition, -1 the + * non-strict one against -128 or 0: the compare against 0 is a test + * without immediate, 127 and -128 fit into an imm8. + */ +static void adjust_cmp_imm(u32 *op, s32 *imm) +{ + if (*imm == 1 || *imm == 128) { + switch (*op) { + case BPF_JLT: + *op = BPF_JLE; + break; + case BPF_JGE: + *op = BPF_JGT; + break; + case BPF_JSLT: + *op = BPF_JSLE; + break; + case BPF_JSGE: + *op = BPF_JSGT; + break; + default: + return; + } + *imm -= 1; + } else if (*imm == -1 || *imm == -129) { + switch (*op) { + case BPF_JGT: + case BPF_JLE: + /* -1 is the largest unsigned value */ + if (*imm == -1) + return; + *op = *op == BPF_JGT ? BPF_JGE : BPF_JLT; + break; + case BPF_JSGT: + *op = BPF_JSGE; + break; + case BPF_JSLE: + *op = BPF_JSLT; + break; + default: + return; + } + *imm += 1; + } +} + +/* + * There is no branch for unwind_goto, only an extable entry + * { .insn = ret_addr - 1, .fixup = target - ret_addr } + * where ret_addr is the return address of the call in front of it. unwind_exit + * replaces the return address of its frame with bpf_unwind_target() of it. + * ret_addr - 1, because what follows the call can have an entry of its own. + * No insn starts there, so a fault cannot hit the entry. + */ +static int add_unwind_exentry(struct bpf_prog *prog, u8 *image, u8 *rw_image, int *excnt, + int ret_addr, int target) +{ + struct exception_table_entry *ex; + s64 delta; + + if (!prog->aux->extable) + return 0; + if (*excnt >= prog->aux->num_exentries) { + pr_err("unwind extable bug\n"); + return -EFAULT; + } + ex = &prog->aux->extable[(*excnt)++]; + delta = image + ret_addr - 1 - (u8 *)&ex->insn; + if (!is_simm32(delta)) { + pr_err("extable->insn doesn't fit into 32-bit\n"); + return -EFAULT; + } + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + ex->insn = delta; + ex->fixup = target - ret_addr; + ex->data = EX_TYPE_BPF; + return 0; +} + +static u64 bpf_unwind_target(u64 ret_addr) +{ + const struct exception_table_entry *e = search_bpf_extables(ret_addr - 1); + + /* + * The verifier made sure that unwind_goto follows a call in a prog. When + * the kernel called an async callback, it's exit. + */ + return e ? ret_addr + e->fixup : ret_addr; +} + +/* unwind_exit: fix the return address of this frame. r0 and r2 are returned. */ +static int emit_unwind_ret_addr(u8 **pprog, u8 *ip) +{ + u8 *prog = *pprog, *start = prog; + + EMIT2(0x50, 0x56); /* push rax; push rsi */ + EMIT4(0x48, 0x8B, 0x7D, 0x08); /* mov rdi, [rbp + 8] */ + ip += prog - start; + ip += x86_call_depth_emit_accounting(&prog, bpf_unwind_target, ip); + if (emit_call(&prog, bpf_unwind_target, ip)) + return -EINVAL; + EMIT4(0x48, 0x89, 0x45, 0x08); /* mov [rbp + 8], rax */ + EMIT2(0x5E, 0x58); /* pop rsi; pop rax */ + *pprog = prog; + return 0; +} + static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { @@ -1738,7 +2585,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; void __percpu *priv_frame_ptr = NULL; u16 out_stack_arg_cnt, outgoing_rsp; - u64 arena_vm_start, user_vm_start; + u64 arena_vm_start = 0, user_vm_start = 0; void __percpu *priv_stack_ptr; int i, excnt = 0; int ilen, proglen = 0; @@ -1778,8 +2625,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * [below] callee-saved regs * [below] outgoing args 7+ (= rsp) */ - arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); - user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + if (bpf_prog_uses_arena_insns(bpf_prog)) { + arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); + user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + } detect_reg_usage(insn, insn_cnt, callee_regs_used); @@ -1860,13 +2709,17 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * const s32 imm32 = insn->imm; u32 dst_reg = insn->dst_reg; u32 src_reg = insn->src_reg; - u8 b2 = 0, b3 = 0; + bool accesses_stack_only; + u8 b3 = 0; u8 *start_of_ldx; s64 jmp_offset; s32 insn_off; + int insn_idx; u8 jmp_cond; u8 *func; int nops; + u32 jmp_op = BPF_OP(insn->code); + s32 cmp_imm; if (priv_frame_ptr) { if (src_reg == BPF_REG_FP) @@ -1880,6 +2733,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_ENDBR(); ip = image + addrs[i - 1] + (prog - temp); + insn_idx = i - 1 + bpf_prog->aux->subprog_start; + accesses_stack_only = + env ? !env->insn_aux_data[insn_idx].non_stack_access : + false; switch (insn->code) { /* ALU */ @@ -1893,43 +2750,24 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_AND | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: - maybe_emit_mod(&prog, dst_reg, src_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - b2 = simple_alu_opcodes[BPF_OP(insn->code)]; - EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); + emit_alu_reg(&prog, BPF_OP(insn->code), dst_reg, src_reg, + BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU64 | BPF_MOV | BPF_X: if (insn_is_cast_user(insn)) { - if (dst_reg != src_reg) - /* 32-bit mov */ - emit_mov_reg(&prog, false, dst_reg, src_reg); - /* shl dst_reg, 32 */ - maybe_emit_1mod(&prog, dst_reg, true); - EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32); - - /* or dst_reg, user_vm_start */ - maybe_emit_1mod(&prog, dst_reg, true); - if (is_axreg(dst_reg)) - EMIT1_off32(0x0D, user_vm_start >> 32); - else - EMIT2_off32(0x81, add_1reg(0xC8, dst_reg), user_vm_start >> 32); - - /* rol dst_reg, 32 */ - maybe_emit_1mod(&prog, dst_reg, true); - EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32); - - /* xor r11, r11 */ - EMIT3(0x4D, 0x31, 0xDB); - - /* test dst_reg32, dst_reg32; check if lower 32-bit are zero */ + /* mov dst_reg32, src_reg32 to zero extend the low half */ + emit_mov_reg(&prog, false, dst_reg, src_reg); + /* mov r11, the high half of user_vm_start; or r11, dst_reg */ + emit_mov_imm64(&prog, AUX_REG, user_vm_start >> 32, 0); + emit_alu_reg(&prog, BPF_OR, AUX_REG, dst_reg, true); + /* test dst_reg32, dst_reg32; a NULL pointer stays NULL */ maybe_emit_mod(&prog, dst_reg, dst_reg, false); EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); - - /* cmove r11, dst_reg; if so, set dst_reg to zero */ + /* cmovne dst_reg, r11 */ /* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */ maybe_emit_mod(&prog, AUX_REG, dst_reg, true); - EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg)); + EMIT3(0x0F, 0x45, add_2reg(0xC0, AUX_REG, dst_reg)); break; } else if (insn_is_mov_percpu_addr(insn)) { /* mov <dst>, <src> (if necessary) */ @@ -1957,9 +2795,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * /* neg dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - EMIT2(0xF7, add_1reg(0xD8, dst_reg)); + emit_neg(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU | BPF_ADD | BPF_K: @@ -1972,42 +2808,23 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_AND | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - - /* - * b3 holds 'normal' opcode, b2 short form only valid - * in case dst is eax/rax. - */ - switch (BPF_OP(insn->code)) { - case BPF_ADD: - b3 = 0xC0; - b2 = 0x05; + if (BPF_OP(insn->code) == BPF_AND && + (imm32 == 0xff || imm32 == 0xffff)) { + /* movzx dst_reg, dst_reg8/16 */ + emit_movzx_reg(&prog, imm32 == 0xff ? 8 : 16, + dst_reg, dst_reg); break; - case BPF_SUB: - b3 = 0xE8; - b2 = 0x2D; - break; - case BPF_AND: - b3 = 0xE0; - b2 = 0x25; - break; - case BPF_OR: - b3 = 0xC8; - b2 = 0x0D; - break; - case BPF_XOR: - b3 = 0xF0; - b2 = 0x35; + } + if (BPF_OP(insn->code) == BPF_XOR && imm32 == -1) { + /* not dst_reg */ + maybe_emit_1mod(&prog, dst_reg, + BPF_CLASS(insn->code) == BPF_ALU64); + EMIT2(0xF7, add_1reg(0xD0, dst_reg)); break; } - if (is_imm8(imm32)) - EMIT3(0x83, add_1reg(b3, dst_reg), imm32); - else if (is_axreg(dst_reg)) - EMIT1_off32(b2, imm32); - else - EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); + emit_alu_imm(&prog, BPF_OP(insn->code), dst_reg, imm32, + BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU64 | BPF_MOV | BPF_K: @@ -2038,6 +2855,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * if (dst_reg != BPF_REG_3) EMIT1(0x52); /* push rdx */ + /* LLVM strength-reduces the 32-bit ones itself */ + if (is64 && BPF_SRC(insn->code) == BPF_K && + emit_div_const(&prog, insn, dst_reg)) + goto div_done; + if (BPF_SRC(insn->code) == BPF_X) { if (src_reg == BPF_REG_0 || src_reg == BPF_REG_3) { @@ -2060,7 +2882,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * xor edx, edx * equivalent to 'xor rdx, rdx', but one byte less */ - EMIT2(0x31, 0xd2); + emit_mov_imm32(&prog, false, BPF_REG_3, 0); /* div src_reg */ maybe_emit_1mod(&prog, src_reg, is64); @@ -2084,7 +2906,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * dst_reg != BPF_REG_0) /* mov dst_reg, rax */ emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); - +div_done: if (dst_reg != BPF_REG_3) EMIT1(0x5A); /* pop rdx */ if (dst_reg != BPF_REG_0) @@ -2094,6 +2916,25 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: + /* + * lea for 3, 5 and 9, which LLVM leaves as a + * multiplication, but not for r13, which needs a + * displacement as base and makes the lea a slow three + * operand form on Intel. + */ + if ((imm32 == 3 || imm32 == 5 || imm32 == 9) && + !base_needs_disp(dst_reg)) { + /* lea dst_reg, [dst_reg + dst_reg * (imm32 - 1)] */ + if (BPF_CLASS(insn->code) == BPF_ALU64) + EMIT1(add_3mod(0x48, dst_reg, dst_reg, dst_reg)); + else if (is_ereg(dst_reg)) + EMIT1(add_3mod(0x40, dst_reg, dst_reg, dst_reg)); + EMIT1(0x8D); + emit_insn_suffix_SIB(&prog, dst_reg, dst_reg, dst_reg, 0, + ilog2(imm32 - 1)); + break; + } + maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); @@ -2124,14 +2965,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - - b3 = simple_alu_opcodes[BPF_OP(insn->code)]; - if (imm32 == 1) - EMIT2(0xD1, add_1reg(b3, dst_reg)); - else - EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); + emit_shift_imm(&prog, BPF_OP(insn->code), dst_reg, + imm32, BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU | BPF_LSH | BPF_X: @@ -2140,8 +2975,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_RSH | BPF_X: case BPF_ALU64 | BPF_ARSH | BPF_X: - /* BMI2 shifts aren't better when shift count is already in rcx */ - if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) { + /* + * BMI2 shifts take the count from any register and, + * unlike the cl forms, are a single uop on Intel, so + * prefer them also when the count already is in rcx. + */ + if (boot_cpu_has(X86_FEATURE_BMI2)) { /* shrx/sarx/shlx dst_reg, dst_reg, src_reg */ bool w = (BPF_CLASS(insn->code) == BPF_ALU64); u8 op; @@ -2197,18 +3036,16 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_END | BPF_FROM_LE: switch (imm32) { case 16: - /* Emit 'ror %ax, 8' to swap lower 2 bytes */ - EMIT1(0x66); + /* Emit 'bswap eax' to swap lower 4 bytes */ if (is_ereg(dst_reg)) EMIT1(0x41); - EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); + EMIT2(0x0F, add_1reg(0xC8, dst_reg)); - /* Emit 'movzwl eax, ax' */ - if (is_ereg(dst_reg)) - EMIT3(0x45, 0x0F, 0xB7); - else - EMIT2(0x0F, 0xB7); - EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); + /* + * Emit 'shr eax, 16' to move the swapped lower + * 2 bytes back down and zero extend + */ + emit_shift_imm(&prog, BPF_RSH, dst_reg, 16, false); break; case 32: /* Emit 'bswap eax' to swap lower 4 bytes */ @@ -2233,11 +3070,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * Emit 'movzwl eax, ax' to zero extend 16-bit * into 64 bit */ - if (is_ereg(dst_reg)) - EMIT3(0x45, 0x0F, 0xB7); - else - EMIT2(0x0F, 0xB7); - EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); + emit_movzx_reg(&prog, 16, dst_reg, dst_reg); break; case 32: /* Emit 'mov eax, eax' to clear upper 32-bits */ @@ -2256,49 +3089,37 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_LFENCE(); break; - /* ST: *(u8*)(dst_reg + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC6); - else - EMIT1(0xC6); - goto st; case BPF_ST | BPF_MEM | BPF_H: - if (is_ereg(dst_reg)) - EMIT3(0x66, 0x41, 0xC7); - else - EMIT2(0x66, 0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_W: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC7); - else - EMIT1(0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_DW: if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { /* Arg 6: store immediate in r9 register */ - emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32); + emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, + imm32); break; } - EMIT2(add_1mod(0x48, dst_reg), 0xC7); - -st: insn_off = insn->off; + insn_off = insn->off; if (dst_reg == BPF_REG_PARAMS) { /* * Args 7+: reverse BPF negative offsets to * x86 positive rsp offsets. * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... */ - insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; + insn_off = outgoing_arg_base - outgoing_rsp - + insn_off - 16; dst_reg = BPF_REG_FP; } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } - EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + emit_st(&prog, insn, dst_reg, insn_off); break; /* STX: *(u8*)(dst_reg + off) = src_reg */ @@ -2316,6 +3137,14 @@ st: insn_off = insn->off; insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; dst_reg = BPF_REG_FP; } + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); break; @@ -2350,28 +3179,8 @@ st: insn_off = insn->off; } populate_extable: { - struct exception_table_entry *ex; - u8 *_insn = image + proglen + (start_of_ldx - temp); u32 arena_reg, fixup_reg; bool is_write; - s64 delta; - - if (!bpf_prog->aux->extable) - break; - - if (excnt >= bpf_prog->aux->num_exentries) { - pr_err("mem32 extable bug\n"); - return -EFAULT; - } - ex = &bpf_prog->aux->extable[excnt++]; - - delta = _insn - (u8 *)&ex->insn; - /* switch ex to rw buffer for writes */ - ex = (void *)rw_image + ((void *)ex - (void *)image); - - ex->insn = delta; - - ex->data = EX_TYPE_BPF; /* * src_reg/dst_reg holds the address in the arena region with upper @@ -2407,14 +3216,12 @@ st: insn_off = insn->off; is_write = true; } - ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, prog - start_of_ldx) | - FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | - FIELD_PREP(FIXUP_REG_MASK, fixup_reg); - ex->fixup |= FIXUP_ARENA_ACCESS; - if (is_write) - ex->fixup |= FIXUP_ARENA_WRITE; - - ex->data |= FIELD_PREP(DATA_ARENA_OFFSET_MASK, insn->off); + err = emit_arena_exentry(bpf_prog, image, rw_image, &excnt, + image + proglen + (start_of_ldx - temp), + prog - start_of_ldx, fixup_reg, + arena_reg, is_write, insn->off); + if (err) + return err; } break; @@ -2455,38 +3262,32 @@ st: insn_off = insn->off; * src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE * and * src_reg + insn->off < VSYSCALL_ADDR + * + * The check is done on src_reg + insn->off - VSYSCALL_ADDR + * against the next power of two above the limit. Addresses + * between the limit and that power of two are non-canonical + * and would fault anyway, so they get zeroed a bit earlier. */ u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR; u8 *end_of_jmp; - /* movabsq r10, VSYSCALL_ADDR */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32, - (u32)(long)VSYSCALL_ADDR); + /* + * lea r11, [src_reg + insn->off - VSYSCALL_ADDR] + * + * VSYSCALL_ADDR is a small negative number, so the + * whole offset fits into the 32-bit displacement. + */ + BUILD_BUG_ON((long)(s32)VSYSCALL_ADDR != (long)VSYSCALL_ADDR); + EMIT2(add_2mod(0x48, src_reg, AUX_REG), 0x8D); + emit_insn_suffix(&prog, src_reg, AUX_REG, + insn->off - (s32)VSYSCALL_ADDR); - /* mov src_reg, r11 */ - EMIT_mov(AUX_REG, src_reg); + /* shr r11, fls64(limit) */ + emit_shift_imm(&prog, BPF_RSH, AUX_REG, fls64(limit), true); - if (insn->off) { - /* add r11, insn->off */ - maybe_emit_1mod(&prog, AUX_REG, true); - EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off); - } - - /* sub r11, r10 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x29, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); - - /* movabsq r10, limit */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, - (u32)(long)limit); - - /* cmp r10, r11 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); - - /* if unsigned '>', goto load */ - EMIT2(X86_JA, 0); + /* if not zero, goto load */ + EMIT2(X86_JNE, 0); end_of_jmp = prog; /* xor dst_reg, dst_reg */ @@ -2494,9 +3295,16 @@ st: insn_off = insn->off; /* jmp byte_after_ldx */ EMIT2(0xEB, 0); - /* populate jmp_offset for JAE above to jump to start_of_ldx */ + /* populate jmp_offset for JNE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; + } else if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, src_reg, + insn_off, + BPF_SIZE(insn->code), ip, + false); + if (err) + return err; } if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX || BPF_MODE(insn->code) == BPF_MEMSX) @@ -2558,68 +3366,52 @@ st: insn_off = insn->off; } fallthrough; case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) { - bool is64 = BPF_SIZE(insn->code) == BPF_DW; - u32 real_src_reg = src_reg; - u32 real_dst_reg = dst_reg; - u8 *branch_target; + case BPF_STX | BPF_ATOMIC | BPF_DW: { + u32 real_src_reg = src_reg; + u32 real_dst_reg = dst_reg; + bool is_atomic_fetch = is_atomic_fetch_op(insn); + u8 *old_prog; - /* - * Can't be implemented with a single x86 insn. - * Need to do a CMPXCHG loop. - */ - + if (is_atomic_fetch) { /* Will need RAX as a CMPXCHG operand so save R0 */ + old_prog = prog; emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); if (src_reg == BPF_REG_0) real_src_reg = BPF_REG_AX; if (dst_reg == BPF_REG_0) real_dst_reg = BPF_REG_AX; - - branch_target = prog; - /* Load old value */ - emit_ldx(&prog, BPF_SIZE(insn->code), - BPF_REG_0, real_dst_reg, insn->off); - /* - * Perform the (commutative) operation locally, - * put the result in the AUX_REG. - */ - emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); - maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64); - EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], - add_2reg(0xC0, AUX_REG, real_src_reg)); - /* Attempt to swap in new value */ - err = emit_atomic_rmw(&prog, BPF_CMPXCHG, - real_dst_reg, AUX_REG, - insn->off, - BPF_SIZE(insn->code)); - if (WARN_ON(err)) + ip += (prog - old_prog); + } + if (!bpf_atomic_is_load_store(insn)) { + if (!accesses_stack_only) { + err = emit_kasan_check( + env, &prog, real_dst_reg, + insn->off, BPF_SIZE(insn->code), + ip, true); + if (err) + return err; + } + } + if (is_atomic_fetch) { + err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, + real_src_reg, -1, false, + NULL, NULL); + if (err) return err; - /* - * ZF tells us whether we won the race. If it's - * cleared we need to try again. - */ - EMIT2(X86_JNE, -(prog - branch_target) - 2); - /* Return the pre-modification value */ - emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0); - /* Restore R0 after clobbering RAX */ - emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); break; } if (bpf_atomic_is_load_store(insn)) - err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg, - insn->off, BPF_SIZE(insn->code)); + err = emit_atomic_ld_st(env, &prog, insn, ip, + dst_reg, src_reg, + accesses_stack_only); else err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg, insn->off, BPF_SIZE(insn->code)); if (err) return err; break; - + } case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: if (!bpf_atomic_is_load_store(insn)) { @@ -2629,6 +3421,53 @@ st: insn_off = insn->off; fallthrough; case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: + if (is_atomic_fetch_op(insn)) { + u32 real_src_reg = src_reg, real_dst_reg = dst_reg; + /* + * An access through a number has its address in + * BPF_REG_AX (see arena_scalar_access()), so R0 is + * saved on the stack instead, as div and mod do. + */ + bool r0_on_stack = dst_reg == BPF_REG_AX; + u8 *fault[2], *resume; + int j; + + /* Will need RAX as a CMPXCHG operand so save R0 */ + if (r0_on_stack) { + EMIT1(0x50); /* push rax */ + } else { + emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); + if (src_reg == BPF_REG_0) + real_src_reg = BPF_REG_AX; + if (dst_reg == BPF_REG_0) + real_dst_reg = BPF_REG_AX; + } + + err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, + real_src_reg, X86_REG_R12, + r0_on_stack, fault, &resume); + if (err) + return err; + + /* + * The page may be unmapped between the load and the + * CMPXCHG, so both need an entry. Both report a write, + * since the BPF insn is a read-modify-write. + */ + for (j = 0; j < 2; j++) { + err = emit_arena_exentry(bpf_prog, image, rw_image, + &excnt, + image + proglen + + (fault[j] - temp), + resume - fault[j], + reg2pt_regs[real_src_reg], + reg2pt_regs[real_dst_reg], + true, insn->off); + if (err) + return err; + } + break; + } start_of_ldx = prog; if (bpf_atomic_is_load_store(insn)) @@ -2645,18 +3484,39 @@ st: insn_off = insn->off; /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { + u8 *tcc_load = prog; + LOAD_TAIL_CALL_CNT_PTR(stack_depth); - ip += 7; + ip += prog - tcc_load; } if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; + ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - + outgoing_rsp, &prog); + } + if (func == (u8 *)arch_bpf_timed_may_goto) { + u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP; + + /* + * AX has the offset of count and timestamp + * in the stack. Turn it into a pointer. + * add r10, rbp or add r10, r9 + */ + maybe_emit_mod(&prog, BPF_REG_AX, fp, true); + EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp)); + ip += 3; } if (priv_frame_ptr) { push_r9(&prog); @@ -2667,9 +3527,63 @@ st: insn_off = insn->off; return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } + /* callx: call of a bpf subprog whose address is in dst_reg */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * The verifier makes sure that callees of callx are + * not tail call reachable, hence unlike a direct call + * of a subprog there is no need to pass + * tail_call_cnt_ptr in rax. + */ + if (priv_frame_ptr) { + push_r9(&prog); + ip += 2; + } + if (emit_indirect_call(&prog, insn->dst_reg, ip)) + return -EINVAL; + if (priv_frame_ptr) + pop_r9(&prog); + break; + + /* unwind_goto. The verifier expanded may_goto. */ + case BPF_JMP | BPF_JCOND: + if (!priv_frame_ptr) { + err = add_unwind_exentry(bpf_prog, image, rw_image, &excnt, + addrs[i - 1], addrs[i + insn->off]); + if (err) + return err; + break; + } + /* + * 'pop r9' after the call has to be done on the way to + * the pad too: + * call foo + * pop r9 <- exit returns here + * jmp 1f + * pop r9 <- unwind_exit returns here + * jmp pad + * 1: + */ + err = add_unwind_exentry(bpf_prog, image, rw_image, &excnt, + addrs[i - 1] - 2, addrs[i - 1] + 2); + if (err) + return err; + EMIT2(0xEB, 7); + pop_r9(&prog); + EMIT1_off32(0xE9, addrs[i + insn->off] - (addrs[i - 1] + 9)); + break; + case BPF_JMP | BPF_TAIL_CALL: if (imm32) emit_bpf_tail_call_direct(bpf_prog, @@ -2725,10 +3639,26 @@ st: insn_off = insn->off; case BPF_JMP | BPF_JSET | BPF_K: case BPF_JMP32 | BPF_JSET | BPF_K: + if ((u32)imm32 <= U8_MAX) { + /* + * test dst_reg8, imm8: with the upper bits of the + * mask clear the result only depends on the low byte. + */ + if (is_ereg_8l(dst_reg)) + EMIT1(add_1mod(0x40, dst_reg)); + if (is_axreg(dst_reg)) + EMIT2(0xA8, imm32); + else + EMIT3(0xF6, add_1reg(0xC0, dst_reg), imm32); + goto emit_cond_jmp; + } /* test dst_reg, imm32 */ maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); - EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); + if (is_axreg(dst_reg)) + EMIT1_off32(0xA9, imm32); + else + EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); goto emit_cond_jmp; case BPF_JMP | BPF_JEQ | BPF_K: @@ -2751,8 +3681,11 @@ st: insn_off = insn->off; case BPF_JMP32 | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: + cmp_imm = imm32; + adjust_cmp_imm(&jmp_op, &cmp_imm); + /* test dst_reg, dst_reg to save one extra byte */ - if (imm32 == 0) { + if (cmp_imm == 0) { maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); @@ -2763,13 +3696,15 @@ st: insn_off = insn->off; maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); - if (is_imm8(imm32)) - EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); + if (is_imm8(cmp_imm)) + EMIT3(0x83, add_1reg(0xF8, dst_reg), cmp_imm); + else if (is_axreg(dst_reg)) + EMIT1_off32(0x3D, cmp_imm); else - EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); + EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), cmp_imm); emit_cond_jmp: /* Convert BPF opcode to x86 */ - switch (BPF_OP(insn->code)) { + switch (jmp_op) { case BPF_JEQ: jmp_cond = X86_JE; break; @@ -2930,13 +3865,16 @@ st: insn_off = insn->off; break; case BPF_JMP | BPF_EXIT: + if (src_reg == BPF_UNWIND_EXIT && bpf_prog_can_unwind(bpf_prog) && + emit_unwind_ret_addr(&prog, ip)) + return -EINVAL; if (seen_exit) { jmp_offset = ctx->cleanup_addr - addrs[i]; goto emit_jmp; } seen_exit = true; /* Update cleanup_addr */ - ctx->cleanup_addr = proglen; + ctx->cleanup_addr = proglen + (prog - temp); if (bpf_prog_was_classic(bpf_prog) && !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) { if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog)) @@ -3051,21 +3989,17 @@ static void clean_stack_garbage(const struct btf_func_model *m, } } -/* get the count of the regs that are used to pass arguments */ -static int get_nr_used_regs(const struct btf_func_model *m) +/* the eightbytes taken by the on-stack arguments, alignment holes included */ +static int get_nr_stack_slots(const u8 *pos_of_slot, int nr_slots) { - int i, arg_regs, nr_used_regs = 0; + int slot; - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_used_regs + arg_regs <= 6) - nr_used_regs += arg_regs; - - if (nr_used_regs >= 6) - break; + for (slot = nr_slots - 1; slot >= 0; slot--) { + if (x86_arg_on_stack(pos_of_slot[slot])) + return x86_arg_stack_off(pos_of_slot[slot]) / 8 + 1; } - return nr_used_regs; + return 0; } /* @@ -3095,20 +4029,21 @@ static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base } static void save_args(const struct btf_func_model *m, u8 **prog, - int stack_size, bool for_call_origin, u32 flags, - u64 arena_base) + const u8 *pos_of_slot, int stack_size, + bool for_call_origin, u32 flags, u64 arena_base) { - int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; + int arg_regs, first_off = 0, nr_stack_slots = 0; bool use_jmp = bpf_trampoline_use_jmp(flags); int stack_args_off = (use_jmp || (flags & BPF_TRAMP_F_INDIRECT)) ? 16 : 24; - int i, j; + int bargs_off = stack_size; + int i, j, slot, off, doff; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: * mov QWORD PTR [rbp-0x10],rdi * mov QWORD PTR [rbp-0x8],rsi */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; @@ -3130,8 +4065,36 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * * the arg1-5,arg7 will be passed by regs, and arg6 will * by stack. + * + * An argument aligned to 16 bytes, an __int128 or a struct + * holding one, also starts at an even stack slot, which can + * leave a hole before it. bpf_jit_place_args() worked out + * where each eightbyte is. */ - if (nr_regs + arg_regs > 6) { + for (j = 0; j < arg_regs; j++, slot++, bargs_off -= 8) { + u8 pos = pos_of_slot[slot]; + u32 src; + + if (!x86_arg_on_stack(pos)) { + /* + * Only copy the arguments on-stack to + * current 'stack_size' and ignore the regs, + * used to prepare the arguments on-stack for + * origin call. + */ + if (for_call_origin) + continue; + + /* copy the arguments from regs into stack */ + src = x86_arg_reg[pos]; + if (arena_arg) { + emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); + src = BPF_REG_0; + } + emit_stx(prog, BPF_DW, BPF_REG_FP, src, -bargs_off); + continue; + } + /* copy function arguments from origin stack frame * into current stack frame. * @@ -3141,53 +4104,33 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * entered through the fentry call, so rbp + 24, and * a single one when it is entered with a jmp or * called indirectly, so rbp + 16. + * + * The area for the origin call mirrors the incoming + * one, hole and all; only the bpf program takes the + * arguments packed. */ - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, - nr_stack_slots * 8 + stack_args_off); - if (arena_arg) - emit_arena_arg_conv(prog, BPF_REG_0, nullable, - (u32)arena_base); - emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, - -stack_size); + off = x86_arg_stack_off(pos); + doff = for_call_origin ? stack_size - off : bargs_off; + emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, + stack_args_off + off); + if (arena_arg) + emit_arena_arg_conv(prog, BPF_REG_0, nullable, + (u32)arena_base); + emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -doff); - if (!nr_stack_slots) - first_off = stack_size; - stack_size -= 8; - nr_stack_slots++; - } - } else { - /* Only copy the arguments on-stack to current - * 'stack_size' and ignore the regs, used to - * prepare the arguments on-stack for origin call. - */ - if (for_call_origin) { - nr_regs += arg_regs; - continue; - } - - /* copy the arguments from regs into stack */ - for (j = 0; j < arg_regs; j++) { - u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs; - - if (arena_arg) { - emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); - src = BPF_REG_0; - } - emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size); - stack_size -= 8; - nr_regs++; - } + if (!nr_stack_slots) + first_off = doff; + nr_stack_slots++; } } clean_stack_garbage(m, prog, nr_stack_slots, first_off); } -static void restore_regs(const struct btf_func_model *m, u8 **prog, +static void restore_regs(u8 **prog, const u8 *pos_of_slot, int nr_slots, int stack_size) { - int i, j, arg_regs, nr_regs = 0; + int slot; /* Restore function arguments from stack. * For a function that accepts two pointers the sequence will be: @@ -3196,23 +4139,10 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, * * The logic here is similar to what we do in save_args() */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_regs + arg_regs <= 6) { - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, - nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, - BPF_REG_FP, - -stack_size); - stack_size -= 8; - nr_regs++; - } - } else { - stack_size -= 8 * arg_regs; - } - - if (nr_regs >= 6) - break; + for (slot = 0; slot < nr_slots; slot++, stack_size -= 8) { + if (!x86_arg_on_stack(pos_of_slot[slot])) + emit_ldx(prog, BPF_DW, x86_arg_reg[pos_of_slot[slot]], + BPF_REG_FP, -stack_size); } } @@ -3401,9 +4331,9 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, return 0; } -/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */ -#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack) \ - __LOAD_TCC_PTR(-round_up(stack, 8) - 8) +/* mov rax, qword ptr [rbp - off] */ +#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(off) \ + __LOAD_TCC_PTR(-(off)) /* Example: * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); @@ -3471,8 +4401,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_nodes *tnodes, void *func_addr) { - int i, ret, nr_regs = m->nr_args, stack_size = 0; + int i, ret, nr_regs, stack_size = 0; + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; int regs_off, func_meta_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; + int tcc_off = 0; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; @@ -3494,12 +4426,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im arena_base = bpf_tramp_arena_base(m, tnodes, flags); - for (i = 0; i < m->nr_args; i++) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - - /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 - * are passed through regs, the remains are through stack. + /* + * x86-64 supports up to MAX_BPF_FUNC_ARGS argument slots, six in + * registers and the rest on the stack, where bpf_jit_place_args() + * puts them. */ + nr_regs = bpf_jit_place_args(&x86_arg_abi, m, pos_of_slot); if (nr_regs > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; @@ -3511,11 +4443,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or * BPF_TRAMP_F_RET_FENTRY_RET flags * - * [ reg_argN ] always - * [ ... ] - * RBP - regs_off [ reg_arg1 ] program's ctx pointer + * [ arg slotN ] always, packed, whether + * [ ... ] in a register or on the + * RBP - regs_off [ arg slot1 ] stack; program's ctx pointer * - * RBP - func_meta_off [ regs count, etc ] always + * RBP - func_meta_off [ slot count, etc ] always * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * @@ -3523,11 +4455,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * * RBP - run_ctx_off [ bpf_tramp_run_ctx ] * - * [ stack_argN ] BPF_TRAMP_F_CALL_ORIG - * [ ... ] - * [ stack_arg2 ] - * RBP - arg_stack_off [ stack_arg1 ] - * RSP [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * RBP - tcc_off [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * + * [ stack slotN ] BPF_TRAMP_F_CALL_ORIG, + * [ ... ] the incoming stack + * [ stack slot2 ] arguments, alignment + * RBP - arg_stack_off [ stack slot1 ] holes included */ /* room for return value of orig_call or fentry prog */ @@ -3538,7 +4471,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += nr_regs * 8; regs_off = stack_size; - /* function matedata, such as regs count */ + /* function matedata, such as the argument slot count */ stack_size += 8; func_meta_off = stack_size; @@ -3558,9 +4491,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; run_ctx_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) { /* the space that used to pass arguments on-stack */ - stack_size += (nr_regs - get_nr_used_regs(m)) * 8; + stack_size += get_nr_stack_slots(pos_of_slot, nr_regs) * 8; /* make sure the stack pointer is 16-byte aligned if we * need pass arguments on stack, which means * [stack_size + 8(rbp) + 8(rip) + 8(origin rip)] @@ -3607,20 +4545,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (im) im->ksym.fp_start = prog - (u8 *)rw_image; - if (!is_imm8(stack_size)) { - /* sub rsp, stack_size */ - EMIT3_off32(0x48, 0x81, 0xEC, stack_size); - } else { - /* sub rsp, stack_size */ - EMIT4(0x48, 0x83, 0xEC, stack_size); - } + /* sub rsp, stack_size */ + emit_sub_rsp(&prog, stack_size); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) - EMIT1(0x50); /* push rax */ + /* mov QWORD PTR [rbp - tcc_off], rax */ + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -tcc_off); /* mov QWORD PTR [rbp - rbx_off], rbx */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); func_meta = nr_regs; - /* Store number of argument registers of the traced function */ + /* Store number of argument slots of the traced function */ emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (flags & BPF_TRAMP_F_IP_ARG) { @@ -3628,7 +4562,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr); } - save_args(m, &prog, regs_off, false, flags, arena_base); + save_args(m, &prog, pos_of_slot, regs_off, false, flags, arena_base); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -3669,14 +4603,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_regs(m, &prog, regs_off); - save_args(m, &prog, arg_stack_off, true, flags, 0); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); + save_args(m, &prog, pos_of_slot, arg_stack_off, true, flags, 0); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } if (flags & BPF_TRAMP_F_ORIG_STACK) { @@ -3723,7 +4657,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_regs(m, &prog, regs_off); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); /* This needs to be done regardless. If there were fmod_ret programs, * the return value is only updated on the stack and still needs to be @@ -3740,7 +4674,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Before running the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } /* restore return value of orig_call or fentry prog back into RAX */ @@ -3755,7 +4689,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip our return address and return to parent */ - EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ + emit_add_rsp(&prog, 8); /* add rsp, 8 */ } emit_return(&prog, image + (prog - (u8 *)rw_image)); /* Make sure the trampoline generation logic doesn't overflow */ @@ -3818,24 +4752,14 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { struct bpf_tramp_image im = {}; - void *image; - int ret; - /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). - * - * We cannot use kvmalloc here, because we need image to be in - * module memory range. - * Since it must be writable use execmem_alloc(EXECMEM_MODULE_DATA) - * that returns writable memory in the module address space. - */ - image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); - if (!image) + if (!trampoline_size_image) return -ENOMEM; - ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image, - m, flags, tnodes, func_addr); - execmem_free(image); - return ret; + return __arch_prepare_bpf_trampoline(&im, trampoline_size_image, + trampoline_size_image + PAGE_SIZE, + trampoline_size_image, m, flags, + tnodes, func_addr); } static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) @@ -4021,6 +4945,22 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr padding = true; goto skip_init_addrs; } + + /* + * An arena fetching AND/OR/XOR is lowered to a CMPXCHG loop whose load + * and CMPXCHG can both fault, one more entry than the verifier reserved. + * The extra pass of a program with subprograms returned above, so the + * count is adjusted once. + */ + for (i = 0; prog->aux->arena && i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + + if (BPF_CLASS(insn->code) == BPF_STX && + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && + is_atomic_fetch_op(insn)) + prog->aux->num_exentries++; + } + addrs = kvmalloc_objs(*addrs, prog->len + 1); if (!addrs) goto out_addrs; @@ -4075,7 +5015,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr * The number of entries in extable is the number of BPF_LDX * insns that access kernel memory via "pointer to BTF type". * The verifier changed their opcode from LDX|MEM|size - * to LDX|PROBE_MEM|size to make JITing easier. + * to LDX|PROBE_MEM|size to make JITing easier. Arena + * accesses add one each, and an arena fetching AND/OR/XOR + * one more, see the adjustment of num_exentries above. */ u32 align = __alignof__(struct exception_table_entry); u32 extable_size = prog->aux->num_exentries * @@ -4159,11 +5101,36 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr return prog; } +bool bpf_jit_supports_callx(void) +{ + /* + * FineIBT poisons ENDBR at the entry of a JITed function and expects + * indirect callers to go through the CFI preamble instead. + * callx doesn't do that yet. + */ + return cfi_mode != CFI_FINEIBT; +} + bool bpf_jit_supports_kfunc_call(void) { return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + +bool bpf_jit_supports_unwind(void) +{ + return true; +} + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &x86_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; @@ -4187,6 +5154,18 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +/* + * Frame sizes are 32-bit immediates in the prologue, epilogue and tail call + * sequences, a tail call pops the caller's frame and lands in the target's + * prologue before the target allocates its own, and private stacks are + * allocated from the program's own depth, so MAX_BPF_STACK_JIT frames need + * nothing special. + */ +bool bpf_jit_supports_large_stack(void) +{ + return true; +} + bool bpf_jit_supports_percpu_insn(void) { return true; @@ -4310,18 +5289,13 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { - if (!in_arena) - return true; - switch (insn->code) { - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) - return false; - } return true; }
diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 54c690c..6c6e03f 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S
@@ -11,12 +11,6 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ANNOTATE_NOENDBR - /* - * r10 passes us stack depth, load the pointer to count and timestamp - * into r10 by adding it to BPF frame pointer. - */ - leaq (%rbp, %r10, 1), %r10 - /* Setup frame. */ pushq %rbp movq %rsp, %rbp @@ -30,8 +24,8 @@ pushq %r8 /* - * r10 has the pointer to count and timestamp, pass it as first - * argument. + * r10 has the pointer to count and timestamp. JIT added the frame + * pointer to the offset. Pass it as first argument. */ movq %r10, %rdi
diff --git a/crypto/Makefile b/crypto/Makefile index 8386d55..33bb5ad 100644 --- a/crypto/Makefile +++ b/crypto/Makefile
@@ -22,9 +22,6 @@ crypto_skcipher-y += skcipher.o obj-$(CONFIG_CRYPTO_SKCIPHER2) += crypto_skcipher.o -ifeq ($(CONFIG_BPF_SYSCALL),y) -obj-$(CONFIG_CRYPTO_SKCIPHER2) += bpf_crypto_skcipher.o -endif obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o obj-$(CONFIG_CRYPTO_ECHAINIV) += echainiv.o
diff --git a/crypto/bpf_crypto_skcipher.c b/crypto/bpf_crypto_skcipher.c deleted file mode 100644 index a88798d..0000000 --- a/crypto/bpf_crypto_skcipher.c +++ /dev/null
@@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2024 Meta, Inc */ -#include <linux/types.h> -#include <linux/module.h> -#include <linux/bpf_crypto.h> -#include <crypto/skcipher.h> - -static void *bpf_crypto_lskcipher_alloc_tfm(const char *algo) -{ - return crypto_alloc_lskcipher(algo, 0, 0); -} - -static void bpf_crypto_lskcipher_free_tfm(void *tfm) -{ - crypto_free_lskcipher(tfm); -} - -static int bpf_crypto_lskcipher_has_algo(const char *algo) -{ - return crypto_has_skcipher(algo, CRYPTO_ALG_TYPE_LSKCIPHER, CRYPTO_ALG_TYPE_MASK); -} - -static int bpf_crypto_lskcipher_setkey(void *tfm, const u8 *key, unsigned int keylen) -{ - return crypto_lskcipher_setkey(tfm, key, keylen); -} - -static u32 bpf_crypto_lskcipher_get_flags(void *tfm) -{ - return crypto_lskcipher_get_flags(tfm); -} - -static unsigned int bpf_crypto_lskcipher_ivsize(void *tfm) -{ - return crypto_lskcipher_ivsize(tfm); -} - -static unsigned int bpf_crypto_lskcipher_statesize(void *tfm) -{ - return crypto_lskcipher_statesize(tfm); -} - -static int bpf_crypto_lskcipher_encrypt(void *tfm, const u8 *src, u8 *dst, - unsigned int len, u8 *siv) -{ - return crypto_lskcipher_encrypt(tfm, src, dst, len, siv); -} - -static int bpf_crypto_lskcipher_decrypt(void *tfm, const u8 *src, u8 *dst, - unsigned int len, u8 *siv) -{ - return crypto_lskcipher_decrypt(tfm, src, dst, len, siv); -} - -static const struct bpf_crypto_type bpf_crypto_lskcipher_type = { - .alloc_tfm = bpf_crypto_lskcipher_alloc_tfm, - .free_tfm = bpf_crypto_lskcipher_free_tfm, - .has_algo = bpf_crypto_lskcipher_has_algo, - .setkey = bpf_crypto_lskcipher_setkey, - .encrypt = bpf_crypto_lskcipher_encrypt, - .decrypt = bpf_crypto_lskcipher_decrypt, - .ivsize = bpf_crypto_lskcipher_ivsize, - .statesize = bpf_crypto_lskcipher_statesize, - .get_flags = bpf_crypto_lskcipher_get_flags, - .owner = THIS_MODULE, - .name = "skcipher", -}; - -static int __init bpf_crypto_skcipher_init(void) -{ - return bpf_crypto_register_type(&bpf_crypto_lskcipher_type); -} - -static void __exit bpf_crypto_skcipher_exit(void) -{ - int err = bpf_crypto_unregister_type(&bpf_crypto_lskcipher_type); - WARN_ON_ONCE(err); -} - -module_init(bpf_crypto_skcipher_init); -module_exit(bpf_crypto_skcipher_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Symmetric key cipher support for BPF");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index a9c6f5d..0e53a02 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -156,6 +156,7 @@ struct amdgpu_watchdog_timer { * Modules parameters. */ extern int amdgpu_modeset; +extern int amdgpu_iommu_perfopt; extern unsigned int amdgpu_vram_limit; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 9269e78..1f67ad0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -33,6 +33,7 @@ #include <linux/console.h> #include <linux/slab.h> #include <linux/iommu.h> +#include <linux/amd-iommu.h> #include <linux/pci.h> #include <linux/pci-p2pdma.h> #include <linux/apple-gmux.h> @@ -1429,16 +1430,15 @@ bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) /** * amdgpu_device_vga_set_decode - enable/disable vga decode * - * @pdev: PCI device pointer + * @data: driver private data (adev) * @state: enable/disable vga decode * * Enable/disable vga decode (all asics). * Returns VGA resource flags. */ -static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev, - bool state) +static unsigned int amdgpu_device_vga_set_decode(void *data, bool state) { - struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev)); + struct amdgpu_device *adev = data; amdgpu_asic_set_vga_state(adev, state); if (state) @@ -3752,6 +3752,28 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) apple_gmux_detect(NULL, NULL))); } +static inline bool amdgpu_device_identity(struct amdgpu_device *adev) +{ + struct pci_dev *pdev = adev->pdev; + struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev); + + if (!domain) + return false; + + return domain->type == IOMMU_DOMAIN_IDENTITY; +} + +static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev) +{ + if (amdgpu_iommu_perfopt == 0) + return false; + + if (!(adev->flags & AMD_IS_APU)) + return false; + + return amdgpu_device_identity(adev); +} + /** * amdgpu_device_init - initialize the driver * @@ -3959,6 +3981,16 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (r) return r; + if (amdgpu_device_use_perfopt(adev)) { + int perfopt_ret = amd_iommu_enable_perfopt(pdev); + + /* Optional optimization; a failure to arm it must not abort probe. */ + if (perfopt_ret) + dev_warn(adev->dev, + "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", + perfopt_ret); + } + /* * No need to remove conflicting FBs for non-display class devices. * This prevents the sysfb from being freed accidently. @@ -4199,7 +4231,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, * ignore it */ if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) - vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); + vga_client_register(adev->pdev, amdgpu_device_vga_set_decode, adev); px = amdgpu_device_supports_px(adev); @@ -4328,6 +4360,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); + if (amdgpu_device_use_perfopt(adev)) + amd_iommu_disable_perfopt(adev->pdev); + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); @@ -4691,6 +4726,20 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; + if (amdgpu_device_use_perfopt(adev)) { + int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev); + + /* + * Must not return on failure: a bare return would leak the + * SR-IOV VF exclusive-mode acquisition taken above (released + * via the exit: path). + */ + if (perfopt_ret) + dev_warn(adev->dev, + "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", + perfopt_ret); + } + if (adev->in_s0ix) amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index d9d794a..d27f767 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -185,6 +185,7 @@ char *amdgpu_disable_cu; char *amdgpu_virtual_display; int amdgpu_enforce_isolation = -1; int amdgpu_modeset = -1; +int amdgpu_iommu_perfopt = -1; /* Specifies the default granularity for SVM, used in buffer * migration and restoration of backing memory when handling @@ -393,6 +394,17 @@ MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)"); module_param_named(aspm, amdgpu_aspm, int, 0444); /** + * DOC: iommu_perfopt (int) + * Control the AMD IOMMU PerfOpt DMA-latency optimization + * (0 = disable; -1 = enable on supported devices). + * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn) + * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it, + * trading isolation for lower DMA latency. + */ +MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)"); +module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444); + +/** * DOC: runpm (int) * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down * the dGPUs when they are idle if supported. The default is -1 (auto enable).
diff --git a/drivers/gpu/drm/i915/display/intel_vga.c b/drivers/gpu/drm/i915/display/intel_vga.c index 4966fd7..4c292fae 100644 --- a/drivers/gpu/drm/i915/display/intel_vga.c +++ b/drivers/gpu/drm/i915/display/intel_vga.c
@@ -283,9 +283,10 @@ void intel_vga_disable(struct intel_display *display) intel_de_posting_read(display, vga_reg); } -static unsigned int intel_vga_set_decode(struct pci_dev *pdev, bool enable_decode) +static unsigned int intel_vga_set_decode(void *data, bool enable_decode) { - struct intel_display *display = to_intel_display(pdev); + struct intel_display *display = data; + struct pci_dev *pdev = to_pci_dev(display->drm->dev); unsigned int decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; drm_dbg_kms(display->drm, "%s VGA decode due to VGA arbitration\n", @@ -331,7 +332,7 @@ void intel_vga_register(struct intel_display *display) * then we do not take part in VGA arbitration and the * vga_client_register() fails with -ENODEV. */ - ret = vga_client_register(pdev, intel_vga_set_decode); + ret = vga_client_register(pdev, intel_vga_set_decode, display); drm_WARN_ON(display->drm, ret && ret != -ENODEV); }
diff --git a/drivers/gpu/drm/loongson/lsdc_drv.c b/drivers/gpu/drm/loongson/lsdc_drv.c index 3fd828d2..5a13df3 100644 --- a/drivers/gpu/drm/loongson/lsdc_drv.c +++ b/drivers/gpu/drm/loongson/lsdc_drv.c
@@ -254,7 +254,7 @@ lsdc_create_device(struct pci_dev *pdev, /* For multiple GPU driver instance co-exixt in the system */ -static unsigned int lsdc_vga_set_decode(struct pci_dev *pdev, bool state) +static unsigned int lsdc_vga_set_decode(void *data, bool state) { return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; } @@ -291,7 +291,7 @@ static int lsdc_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, ddev); - vga_client_register(pdev, lsdc_vga_set_decode); + vga_client_register(pdev, lsdc_vga_set_decode, NULL); drmm_kms_helper_poll_init(ddev);
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c index a6c375a..17464c8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vga.c +++ b/drivers/gpu/drm/nouveau/nouveau_vga.c
@@ -9,9 +9,9 @@ #include "nouveau_vga.h" static unsigned int -nouveau_vga_set_decode(struct pci_dev *pdev, bool state) +nouveau_vga_set_decode(void *data, bool state) { - struct nouveau_drm *drm = pci_get_drvdata(pdev); + struct nouveau_drm *drm = data; struct nvif_object *device = &drm->client.device.object; if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CURIE && @@ -95,7 +95,7 @@ nouveau_vga_init(struct nouveau_drm *drm) return; pdev = to_pci_dev(dev->dev); - vga_client_register(pdev, nouveau_vga_set_decode); + vga_client_register(pdev, nouveau_vga_set_decode, drm); /* don't register Thunderbolt eGPU with vga_switcheroo */ if (pci_is_thunderbolt_attached(pdev))
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 705c012..d21fdec 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1070,16 +1070,15 @@ void radeon_combios_fini(struct radeon_device *rdev) /** * radeon_vga_set_decode - enable/disable vga decode * - * @pdev: PCI device + * @data: driver private data (rdev) * @state: enable/disable vga decode * * Enable/disable vga decode (all asics). * Returns VGA resource flags. */ -static unsigned int radeon_vga_set_decode(struct pci_dev *pdev, bool state) +static unsigned int radeon_vga_set_decode(void *data, bool state) { - struct drm_device *dev = pci_get_drvdata(pdev); - struct radeon_device *rdev = dev->dev_private; + struct radeon_device *rdev = data; radeon_vga_set_state(rdev, state); if (state) return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | @@ -1424,7 +1423,7 @@ int radeon_device_init(struct radeon_device *rdev, /* if we have > 1 VGA cards, then disable the radeon VGA resources */ /* this will fail for cards that aren't VGA class devices, just * ignore it */ - vga_client_register(rdev->pdev, radeon_vga_set_decode); + vga_client_register(rdev->pdev, radeon_vga_set_decode, rdev); if (rdev->flags & RADEON_IS_PX) runtime = true;
diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index 31256cb..4f3bb9b 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c
@@ -21,6 +21,7 @@ #include <linux/panic_notifier.h> #include <linux/ptrace.h> #include <linux/random.h> +#include <linux/rhashtable.h> #include <linux/efi.h> #include <linux/kdebug.h> #include <linux/kmsg_dump.h> @@ -78,6 +79,22 @@ static struct ctl_table_header *hv_ctl_table_hdr; u8 * __percpu *hv_synic_eventring_tail; EXPORT_SYMBOL_GPL(hv_synic_eventring_tail); +struct hv_pci_busdata { + int pci_domain_nr; + u32 logical_dev_id_prefix; + struct rhash_head node; + struct rcu_head rcu; +}; + +static struct rhashtable hv_pci_bus_ht; +static bool hv_pci_bus_ht_initialized; + +static const struct rhashtable_params hv_pci_bus_ht_params = { + .key_len = sizeof_field(struct hv_pci_busdata, pci_domain_nr), + .key_offset = offsetof(struct hv_pci_busdata, pci_domain_nr), + .head_offset = offsetof(struct hv_pci_busdata, node), +}; + /* * Hyper-V specific initialization and shutdown code that is * common across all architectures. Called from architecture @@ -86,6 +103,11 @@ EXPORT_SYMBOL_GPL(hv_synic_eventring_tail); void __init hv_common_free(void) { + if (hv_pci_bus_ht_initialized) { + rhashtable_destroy(&hv_pci_bus_ht); + hv_pci_bus_ht_initialized = false; + } + unregister_sysctl_table(hv_ctl_table_hdr); hv_ctl_table_hdr = NULL; @@ -315,6 +337,7 @@ u8 __init get_vtl(void) int __init hv_common_init(void) { int i; + int ret; union hv_hypervisor_version_info version; /* Get information about the Microsoft Hypervisor version */ @@ -394,6 +417,13 @@ int __init hv_common_init(void) for (i = 0; i < nr_cpu_ids; i++) hv_vp_index[i] = VP_INVAL; + ret = rhashtable_init(&hv_pci_bus_ht, &hv_pci_bus_ht_params); + if (ret) { + hv_common_free(); + return ret; + } + hv_pci_bus_ht_initialized = true; + return 0; } @@ -808,6 +838,7 @@ static const struct hv_status_info hv_status_infos[] = { _STATUS_INFO(HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED, -EOPNOTSUPP), _STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EINVAL), _STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EINVAL), + _STATUS_INFO(HV_STATUS_DEVICE_ALREADY_IN_DOMAIN, -EEXIST), _STATUS_INFO(HV_STATUS_OPERATION_FAILED, -EIO), _STATUS_INFO(HV_STATUS_TIME_OUT, -EIO), _STATUS_INFO(HV_STATUS_CALL_PENDING, -EIO), @@ -861,3 +892,87 @@ const char *hv_result_to_string(u64 status) return "Unknown"; } EXPORT_SYMBOL_GPL(hv_result_to_string); + +/* + * Logical device ID registry for Hyper-V PCI buses. The pci-hyperv + * driver registers each bus's logical device ID prefix before scanning + * its devices. Consumers look up the prefix by PCI domain number when + * building logical device IDs for Hyper-V interfaces. + */ +int hv_pci_register_dev_id(int pci_domain_nr, u32 logical_dev_id_prefix) +{ + struct hv_pci_busdata *bus, *new; + int ret; + + new = kzalloc_obj(*new, GFP_KERNEL); + if (!new) + return -ENOMEM; + + new->pci_domain_nr = pci_domain_nr; + new->logical_dev_id_prefix = logical_dev_id_prefix; + + bus = rhashtable_lookup_get_insert_fast(&hv_pci_bus_ht, &new->node, + hv_pci_bus_ht_params); + if (IS_ERR(bus)) { + ret = PTR_ERR(bus); + goto free_new; + } + + if (WARN_ONCE(bus != NULL, + "Hyper-V PCI domain %04x is already registered\n", + pci_domain_nr)) { + ret = -EEXIST; + goto free_new; + } + + return 0; + +free_new: + kfree(new); + return ret; +} +EXPORT_SYMBOL_FOR_MODULES(hv_pci_register_dev_id, "pci-hyperv"); + +void hv_pci_unregister_dev_id(int pci_domain_nr) +{ + struct hv_pci_busdata *bus; + int ret = -ENOENT; + + rcu_read_lock(); + bus = rhashtable_lookup(&hv_pci_bus_ht, &pci_domain_nr, + hv_pci_bus_ht_params); + if (bus) + ret = rhashtable_remove_fast(&hv_pci_bus_ht, &bus->node, + hv_pci_bus_ht_params); + rcu_read_unlock(); + + if (WARN_ONCE(ret, + "Failed to unregister Hyper-V PCI domain %04x\n", + pci_domain_nr)) + return; + + kfree_rcu(bus, rcu); +} +EXPORT_SYMBOL_FOR_MODULES(hv_pci_unregister_dev_id, "pci-hyperv"); + +/* + * Look up the logical device ID prefix registered for @pci_domain_nr. + * Returns 0 on success with *prefix filled in; -ENODEV if no entry is + * registered for that PCI domain. + */ +int hv_pci_lookup_dev_id(int pci_domain_nr, u32 *prefix) +{ + struct hv_pci_busdata *bus; + int ret = -ENODEV; + + rcu_read_lock(); + bus = rhashtable_lookup(&hv_pci_bus_ht, &pci_domain_nr, + hv_pci_bus_ht_params); + if (bus) { + *prefix = bus->logical_dev_id_prefix; + ret = 0; + } + rcu_read_unlock(); + + return ret; +}
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6e07bd6..85a8d7f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig
@@ -195,6 +195,7 @@ source "drivers/iommu/amd/Kconfig" source "drivers/iommu/arm/Kconfig" source "drivers/iommu/intel/Kconfig" +source "drivers/iommu/hyperv/Kconfig" source "drivers/iommu/iommufd/Kconfig" source "drivers/iommu/riscv/Kconfig" @@ -303,7 +304,7 @@ config APPLE_DART tristate "Apple DART IOMMU Support" - depends on ARCH_APPLE || COMPILE_TEST + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_DART select IOMMU_API select IOMMU_IO_PGTABLE_DART @@ -386,6 +387,21 @@ Say Y here if you want to use this IOMMU in front of these hardware blocks. +config BCM2712_IOMMU + bool "BCM2712 IOMMU driver" + depends on (ARCH_BCM2835 && ARM64) || COMPILE_TEST + select IOMMU_API + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_BCM2712 + help + Support for IOMMU on BCM2712 SoC. This IOMMU can be used by the + display controller and various multimedia devices to perform + efficient memory management. + + Say Y here if you want to use this IOMMU in front of these + hardware blocks. + config IOMMU_DEBUG_PAGEALLOC bool "Debug IOMMU mappings against page allocations" depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 2f05725..29a26d2 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile
@@ -37,4 +37,5 @@ obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o +obj-$(CONFIG_BCM2712_IOMMU) += bcm2712-iommu.o bcm2712-iommu-cache.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 71113e86..93845b3 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h
@@ -48,6 +48,9 @@ extern u8 amd_iommu_hpt_vasize; extern unsigned long amd_iommu_pgsize_bitmap; extern bool amd_iommu_hatdis; +int amd_iommu_perfopt_clear(struct amd_iommu *iommu); +int amd_iommu_perfopt_restore(struct amd_iommu *iommu); + /* Protection domain ops */ void amd_iommu_init_identity_domain(void); struct protection_domain *protection_domain_alloc(void);
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 4a15b16..0e79c43 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -53,6 +53,7 @@ #define MMIO_MSI_ADDR_LO_OFFSET 0x015C #define MMIO_MSI_ADDR_HI_OFFSET 0x0160 #define MMIO_MSI_DATA_OFFSET 0x0164 +#define MMIO_PERF_OPT_OFFSET 0x016C #define MMIO_INTCAPXT_EVT_OFFSET 0x0170 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180 @@ -87,6 +88,8 @@ #define FEATURE_GLX GENMASK_ULL(15, 14) #define FEATURE_GAM_VAPIC BIT_ULL(21) #define FEATURE_PASMAX GENMASK_ULL(36, 32) +#define FEATURE_PERF_OPT BIT_ULL(45) +#define PERF_OPT_EN BIT(13) #define FEATURE_GIOSUP BIT_ULL(48) #define FEATURE_HASUP BIT_ULL(49) #define FEATURE_EPHSUP BIT_ULL(50) @@ -657,6 +660,9 @@ struct amd_iommu { /* Extended features 2 */ u64 features2; + /* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */ + int perfopt_refcount; + /* PCI device id of the IOMMU device */ u16 devid; @@ -818,6 +824,7 @@ struct iommu_dev_data { u8 ppr :1; /* Enable device PPR support */ bool use_vapic; /* Enable device to use vapic mode */ bool defer_attach; + bool perfopt; struct ratelimit_state rs; /* Ratelimit IOPF messages */ };
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index edcc187..de6db56 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c
@@ -163,7 +163,6 @@ int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; -static bool amd_iommu_detected; static bool amd_iommu_disabled __initdata; static bool amd_iommu_force_enable __initdata; static bool amd_iommu_irtcachedis; @@ -390,7 +389,13 @@ static void iommu_set_device_table(struct amd_iommu *iommu) BUG_ON(iommu->mmio_base == NULL); - if (is_kdump_kernel()) + /* + * A kdump kernel that reuses the previous kernel's device table must + * leave the base register alone. It already points at that table, and + * with SNP enabled the register is locked. If reuse failed, program + * the freshly allocated table like a normal boot. + */ + if (iommu->pci_seg->old_dev_tbl_cpy) return; entry = iommu_virt_to_phys(dev_table); @@ -623,8 +628,19 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_ /* Allocate per PCI segment device table */ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->dev_table = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, - pci_seg->dev_table_size); + gfp_t gfp = GFP_KERNEL; + + /* + * Keep normal kernel device tables below 4G because a later kdump + * kernel only trusts and reuses old tables below that limit. A kdump + * kernel's new table is either discarded when reuse succeeds or + * programmed after the IOMMU is disabled, so the table does not + * need DMA32 memory (which the crashkernel may not have). + */ + if (!is_kdump_kernel()) + gfp |= GFP_DMA32; + + pci_seg->dev_table = iommu_alloc_pages_sz(gfp, pci_seg->dev_table_size); if (!pci_seg->dev_table) return -ENOMEM; @@ -916,13 +932,13 @@ static void free_ga_log(struct amd_iommu *iommu) } #ifdef CONFIG_IRQ_REMAP -static int iommu_ga_log_enable(struct amd_iommu *iommu) +static bool iommu_ga_log_enable(struct amd_iommu *iommu) { u32 status, i; u64 entry; if (!iommu->ga_log) - return -EINVAL; + return false; entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, @@ -946,35 +962,31 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) } if (WARN_ON(i >= MMIO_STATUS_TIMEOUT)) - return -EINVAL; + return false; - return 0; + return true; } +#endif /* CONFIG_IRQ_REMAP */ -static int iommu_init_ga_log(struct amd_iommu *iommu) +static int alloc_ga_log(struct amd_iommu *iommu) { +#ifdef CONFIG_IRQ_REMAP int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; - if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) - return -EINVAL; - - if (iommu->ga_log && iommu->ga_log_tail) - return 0; - iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE); if (!iommu->ga_log) - goto err_out; + return -ENOMEM; iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8); - if (!iommu->ga_log_tail) - goto err_out; + if (!iommu->ga_log_tail) { + iommu_free_pages(iommu->ga_log); + iommu->ga_log = NULL; + return -ENOMEM; + } +#endif return 0; -err_out: - free_ga_log(iommu); - return -EINVAL; } -#endif /* CONFIG_IRQ_REMAP */ static int __init alloc_cwwb_sem(struct amd_iommu *iommu) { @@ -1948,6 +1960,9 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, if (!iommu->mmio_base) return -ENOMEM; + if (amd_iommu_perfopt_clear(iommu)) + pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index); + return init_iommu_from_acpi(iommu, h); } @@ -2152,6 +2167,27 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu) } } +/* Must be called after SNP support check is complete (iommu_snp_enable()) */ +static bool check_vapic_support(void) +{ + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + return false; + + if (!check_feature(FEATURE_GAM_VAPIC)) { + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; + return false; + } + + if (amd_iommu_snp_en && + !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) { + pr_warn("Force to disable Virtual APIC due to SNP\n"); + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; + return false; + } + + return true; +} + static int __init iommu_init_pci(struct amd_iommu *iommu) { int cap_ptr = iommu->cap_ptr; @@ -2196,6 +2232,12 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) return -ENOMEM; + if (check_vapic_support()) { + ret = alloc_ga_log(iommu); + if (ret) + return ret; + } + if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) { pr_info("Using strict mode due to virtualization\n"); iommu_set_dma_strict(); @@ -3010,22 +3052,9 @@ static void enable_iommus_vapic(void) if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return; - if (!check_feature(FEATURE_GAM_VAPIC)) { - amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; - return; - } - - if (amd_iommu_snp_en && - !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) { - pr_warn("Force to disable Virtual APIC due to SNP\n"); - amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; - return; - } - /* Enabling GAM and SNPAVIC support */ for_each_iommu(iommu) { - if (iommu_init_ga_log(iommu) || - iommu_ga_log_enable(iommu)) + if (!iommu_ga_log_enable(iommu)) return; iommu_feature_enable(iommu, CONTROL_GAM_EN); @@ -3038,10 +3067,46 @@ static void enable_iommus_vapic(void) #endif } +static int clear_perfopt_all(void) +{ + struct amd_iommu *iommu; + int err, ret = 0; + + for_each_iommu(iommu) { + err = amd_iommu_perfopt_clear(iommu); + if (err) + ret = err; + } + + return ret; +} + +static int restore_perfopt_all(void) +{ + struct amd_iommu *iommu; + int err, ret = 0; + + for_each_iommu(iommu) { + err = amd_iommu_perfopt_restore(iommu); + if (err) + ret = err; + } + + return ret; +} + static void disable_iommus(void) { struct amd_iommu *iommu; + /* + * PerfOpt is an optional performance bit, so a failure to clear it must + * not skip the mandatory disable below. This also runs from the void + * amd_iommu_disable() shutdown/kexec path, which cannot report an error. + */ + if (clear_perfopt_all()) + pr_err("Failed to clear PerfOpt while disabling IOMMUs\n"); + for_each_iommu(iommu) iommu_disable(iommu); @@ -3067,6 +3132,10 @@ static void amd_iommu_resume(void *data) for_each_iommu(iommu) early_enable_iommu(iommu); + /* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */ + if (restore_perfopt_all()) + pr_err("Failed to restore PerfOpt after IOMMU resume\n"); + iommu_enable_event_buffer(); amd_iommu_enable_interrupts(); } @@ -3228,9 +3297,6 @@ static int __init early_amd_iommu_init(void) acpi_status status; u8 efr_hats, max_vasize; - if (!amd_iommu_detected) - return -ENODEV; - status = acpi_get_table("IVRS", 0, &ivrs_base); if (status == AE_NOT_FOUND) return -ENODEV; @@ -3309,7 +3375,7 @@ static int __init early_amd_iommu_init(void) } /* Disable any previously enabled IOMMUs */ - if (!is_kdump_kernel() || amd_iommu_disabled) + if (!is_kdump_kernel()) disable_iommus(); if (amd_iommu_irq_remap) @@ -3404,12 +3470,6 @@ static __init void iommu_snp_enable(void) if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - /* SNP support required IOMMU to be ON */ - if (no_iommu) { - pr_warn("SNP: IOMMU disabled, SNP cannot be supported.\n"); - goto disable_snp; - } - amd_iommu_snp_mode0_sup = check_feature2(FEATURE_SNP_PAGE_MODE0_SUP); /* * If SNP page mode 0 is not enabled, then SNP support requires that IOMMU @@ -3476,6 +3536,8 @@ static void amd_iommu_apply_erratum_snp(void) #endif } +static bool amd_iommu_sme_check(void); + /**************************************************************************** * * AMD IOMMU Initialization State Machine @@ -3488,7 +3550,13 @@ static int __init state_next(void) switch (init_state) { case IOMMU_START_STATE: - if (!detect_ivrs()) { + if (no_iommu || amd_iommu_disabled) { + init_state = IOMMU_CMDLINE_DISABLED; + ret = -EINVAL; + } else if (!amd_iommu_sme_check()) { + init_state = IOMMU_INIT_ERROR; + ret = -EINVAL; + } else if (!detect_ivrs()) { init_state = IOMMU_NOT_FOUND; ret = -ENODEV; } else { @@ -3496,13 +3564,8 @@ static int __init state_next(void) } break; case IOMMU_IVRS_DETECTED: - if (amd_iommu_disabled) { - init_state = IOMMU_CMDLINE_DISABLED; - ret = -EINVAL; - } else { - ret = early_amd_iommu_init(); - init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; - } + ret = early_amd_iommu_init(); + init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; break; case IOMMU_ACPI_FINISHED: early_enable_iommus(); @@ -3574,6 +3637,19 @@ static int __init iommu_go_to_state(enum iommu_init_state state) { int ret = -EINVAL; + /* + * Some essential housekeeping work is done by amd_iommu_detect(). + * Skipping calling it implies that the platform (e.g., Xen hypervisor) + * has taken over the hardware. Progressing the state machine in this + * case is worthless and fragile. + * + * There are several paths requesting later states, so disallow implicit + * START_STATE => IVRS_DETECTED transition to prevent these paths from + * accidentally progressing the state machine. + */ + if (init_state == IOMMU_START_STATE && state != IOMMU_IVRS_DETECTED) + goto out; + while (init_state != state) { if (init_state == IOMMU_NOT_FOUND || init_state == IOMMU_INIT_ERROR || @@ -3582,6 +3658,7 @@ static int __init iommu_go_to_state(enum iommu_init_state state) ret = state_next(); } +out: /* * SNP platform initilazation requires IOMMUs to be fully configured. * If the SNP support on IOMMUs has NOT been checked, simply mark SNP @@ -3701,17 +3778,10 @@ void __init amd_iommu_detect(void) { int ret; - if (no_iommu || (iommu_detected && !gart_iommu_aperture)) - goto disable_snp; - - if (!amd_iommu_sme_check()) - goto disable_snp; - ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); if (ret) goto disable_snp; - amd_iommu_detected = true; iommu_detected = 1; x86_init.iommu.iommu_init = amd_iommu_init; return;
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 893cd2b..960e1dc 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c
@@ -50,8 +50,6 @@ #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) /* Reserved IOVA ranges */ -#define MSI_RANGE_START (0xfee00000) -#define MSI_RANGE_END (0xfeefffff) #define HT_RANGE_START (0xfd00000000ULL) #define HT_RANGE_END (0xffffffffffULL) @@ -573,10 +571,17 @@ static inline int pdev_enable_cap_ats(struct pci_dev *pdev) if (amd_iommu_iotlb_sup && (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) { ret = pci_enable_ats(pdev, PAGE_SHIFT); - if (!ret) { - dev_data->ats_enabled = 1; - dev_data->ats_qdep = pci_ats_queue_depth(pdev); - } + + /* + * pci_enable_ats() should not fail here because earlier + * checks have already verified support & config. + */ + if (WARN_ON(ret)) + return ret; + + dev_data->ats_enabled = 1; + dev_data->ats_qdep = pci_ats_queue_depth(pdev); + ret = 0; } return ret; @@ -675,7 +680,8 @@ static void pdev_disable_caps(struct pci_dev *pdev) * This function checks if the driver got a valid device from the caller to * avoid dereferencing invalid pointers. */ -static bool check_device(struct device *dev) +static bool lookup_device(struct device *dev, + struct amd_iommu **iommu_out, u16 *devid_out) { struct amd_iommu_pci_seg *pci_seg; struct amd_iommu *iommu; @@ -690,7 +696,7 @@ static bool check_device(struct device *dev) devid = PCI_SBDF_TO_DEVID(sbdf); iommu = rlookup_amd_iommu(dev); - if (!iommu) + if (!iommu || !iommu->iommu.ops) return false; /* Out of our scope? */ @@ -698,65 +704,35 @@ static bool check_device(struct device *dev) if (devid > pci_seg->last_bdf) return false; + *iommu_out = iommu; + *devid_out = devid; return true; } -static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) +static struct iommu_dev_data *iommu_init_device(struct amd_iommu *iommu, + struct device *dev, u16 devid) { struct iommu_dev_data *dev_data; - int devid, sbdf; - if (dev_iommu_priv_get(dev)) - return 0; - - sbdf = get_device_sbdf_id(dev); - if (sbdf < 0) - return sbdf; - - devid = PCI_SBDF_TO_DEVID(sbdf); dev_data = find_dev_data(iommu, devid); if (!dev_data) - return -ENOMEM; + return ERR_PTR(-ENOMEM); dev_data->dev = dev; /* - * The dev_iommu_priv_set() needes to be called before setup_aliases. + * The dev_iommu_priv_set() needs to be called before setup_aliases. * Otherwise, subsequent call to dev_iommu_priv_get() will fail. */ dev_iommu_priv_set(dev, dev_data); setup_aliases(iommu, dev); - /* - * By default we use passthrough mode for IOMMUv2 capable device. - * But if amd_iommu=force_isolation is set (e.g. to debug DMA to - * invalid address), we ignore the capability for the device so - * it'll be forced to go into translation mode. - */ - if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && - dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) { - dev_data->flags = pdev_get_caps(to_pci_dev(dev)); - } + /* Wait for DTE updates to go through */ + iommu_completion_wait(iommu); - return 0; + return dev_data; } -static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) -{ - struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; - struct dev_table_entry *dev_table = get_dev_table(iommu); - int devid, sbdf; - - sbdf = get_device_sbdf_id(dev); - if (sbdf < 0) - return; - - devid = PCI_SBDF_TO_DEVID(sbdf); - pci_seg->rlookup_table[devid] = NULL; - memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); - - setup_aliases(iommu, dev); -} /**************************************************************************** @@ -2392,6 +2368,9 @@ static int attach_device(struct device *dev, if (ret) goto out; + if (dev_data->perfopt) + goto skip_caps; + /* Setup GCR3 table */ if (pdom_is_sva_capable(domain)) { ret = init_gcr3_table(dev_data, domain); @@ -2416,6 +2395,7 @@ static int attach_device(struct device *dev, pdev_enable_cap_ats(pdev); } +skip_caps: /* Update data structures */ dev_data->domain = domain; spin_lock_irqsave(&domain->lock, flags); @@ -2484,73 +2464,289 @@ static void detach_device(struct device *dev) mutex_unlock(&dev_data->mutex); } -static struct iommu_device *amd_iommu_probe_device(struct device *dev) +static int iommu_init_device_caps(struct iommu_dev_data *dev_data, + struct device *dev, + struct amd_iommu *iommu) { - struct iommu_device *iommu_dev; - struct amd_iommu *iommu; - struct iommu_dev_data *dev_data; int ret; - if (!check_device(dev)) - return ERR_PTR(-ENODEV); - - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return ERR_PTR(-ENODEV); - - /* Not registered yet? */ - if (!iommu->iommu.ops) - return ERR_PTR(-ENODEV); - - if (dev_iommu_priv_get(dev)) - return &iommu->iommu; - - ret = iommu_init_device(iommu, dev); - if (ret) { - dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); - iommu_dev = ERR_PTR(ret); - iommu_ignore_device(iommu, dev); - goto out_err; - } - - amd_iommu_set_pci_msi_domain(dev, iommu); - iommu_dev = &iommu->iommu; - - /* - * If IOMMU and device supports PASID then it will contain max - * supported PASIDs, else it will be zero. - */ - dev_data = dev_iommu_priv_get(dev); - if (amd_iommu_pasid_supported() && dev_is_pci(dev) && - pdev_pasid_supported(dev_data)) { - dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, - pci_max_pasids(to_pci_dev(dev))); - } - - if (amd_iommu_pgtable == PD_MODE_NONE) { - pr_warn_once("%s: DMA translation not supported by iommu.\n", - __func__); - iommu_dev = ERR_PTR(-ENODEV); - goto out_err; - } - - iommu_completion_wait(iommu); - if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; else dev_data->max_irqs = MAX_IRQS_PER_TABLE_512; - if (dev_is_pci(dev)) - pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); + amd_iommu_set_pci_msi_domain(dev, iommu); -out_err: + if (!dev_is_pci(dev)) + return 0; + + /* + * By default we use passthrough mode for IOMMUv2 capable device. + * But if amd_iommu=force_isolation is set (e.g. to debug DMA to + * invalid address), we ignore the capability for the device so + * it'll be forced to go into translation mode. + */ + if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && + amd_iommu_gt_ppr_supported()) { + dev_data->flags = pdev_get_caps(to_pci_dev(dev)); + } + + /* + * If IOMMU and device supports PASID then it will contain max + * supported PASIDs, else it will be zero. + */ + if (amd_iommu_pasid_supported() && + pdev_pasid_supported(dev_data)) { + dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, + pci_max_pasids(to_pci_dev(dev))); + } + + if (pci_ats_supported(to_pci_dev(dev))) { + ret = pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); + if (ret) + return ret; + } + + return 0; +} + +/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */ +static int __perfopt_write(struct amd_iommu *iommu, bool enable) +{ + u32 old, val, readback; + + if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) + return enable ? -ENODEV : 0; + + old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (old == U32_MAX) + return -EIO; + + val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; + if (val != old) + writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (readback == U32_MAX || + (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) + return -EIO; + return 0; +} + +/* + * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is + * reference counted: armed on the first requesting device, cleared on the last. + */ +static int perfopt_get(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret = 0; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + if (iommu->perfopt_refcount == 0) { + ret = __perfopt_write(iommu, true); + if (ret) + goto out; + } + iommu->perfopt_refcount++; +out: + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +static int perfopt_put(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret = 0; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0) + ret = __perfopt_write(iommu, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* + * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown, + * and suspend). The count is preserved so amd_iommu_perfopt_restore() can + * re-arm on resume. + */ +int amd_iommu_perfopt_clear(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __perfopt_write(iommu, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* + * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on + * resume, so devices armed before suspend keep the optimization without each + * consumer driver re-arming. + */ +int amd_iommu_perfopt_restore(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); + struct protection_domain *domain; + int ret; + + if (!iommu || !dev_data) + return -ENODEV; + + if (!(iommu->features & FEATURE_PERF_OPT)) + return -ENODEV; + + domain = dev_data->domain; + if (!domain) + return -ENODEV; + + /* Already armed for this device (e.g. re-entry on resume). */ + if (dev_data->perfopt) + return 0; + + /* + * The bit is only architecturally valid while the device is untranslated: + * identity domain with ATS/PRI/PASID off. The identity domain is + * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3 + * table. Re-home the device onto the same identity domain with + * perfopt set, so the attach_device() skip_caps path leaves + * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach + * pattern used by amd_iommu_attach_device(). + * + * Locking: this and amd_iommu_disable_perfopt() run only from the + * consumer driver's bind/unbind path. group->mutex is not exposed to + * drivers, but a device bound to its native driver cannot have its domain + * changed concurrently by the core (VFIO ownership is mutually exclusive; + * sysfs domain changes require an unused group), so the detach/attach pair + * is serialized without it. + */ + dev_data->perfopt = true; + detach_device(&pdev->dev); + ret = attach_device(&pdev->dev, domain); + if (ret) + goto err_restore; + + ret = perfopt_get(iommu); + if (ret) + goto err_rearm; + + dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index); + return 0; + +err_rearm: + detach_device(&pdev->dev); +err_restore: + dev_data->perfopt = false; + if (attach_device(&pdev->dev, domain)) + pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n"); + dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n", + iommu->index, ret); + return ret; +} +EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt); + +void amd_iommu_disable_perfopt(struct pci_dev *pdev) +{ + struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); + struct protection_domain *domain; + + if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain) + return; + + if (WARN_ON(perfopt_put(iommu))) + pci_err(pdev, "failed to clear PerfOpt\n"); + + /* + * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its + * identity domain with the flag cleared, so a later bind without PerfOpt + * sees a normally-capable device. See the locking note in + * amd_iommu_enable_perfopt(). + */ + domain = dev_data->domain; + dev_data->perfopt = false; + detach_device(&pdev->dev); + if (attach_device(&pdev->dev, domain)) + pci_err(pdev, "failed to restore caps after PerfOpt disable\n"); +} +EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt); +static struct iommu_device *amd_iommu_probe_device(struct device *dev) +{ + struct iommu_device *iommu_dev; + struct amd_iommu *iommu; + struct iommu_dev_data *dev_data; + u16 devid; + int ret; + + if (!lookup_device(dev, &iommu, &devid)) + return ERR_PTR(-ENODEV); + + if (dev_iommu_priv_get(dev)) + return &iommu->iommu; + + dev_data = iommu_init_device(iommu, dev, devid); + if (IS_ERR(dev_data)) { + dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); + return ERR_CAST(dev_data); + } + + ret = iommu_init_device_caps(dev_data, dev, iommu); + if (ret) + return ERR_PTR(ret); + + iommu_dev = &iommu->iommu; + + /* + * When DMA translation is unavailable return error so the iommu core + * won't attempt domain attach for this device, while preserving its + * rlookup entry for interrupt remapping. + */ + if (amd_iommu_pgtable == PD_MODE_NONE) { + pr_warn_once("%s: DMA translation not supported by iommu.\n", + __func__); + return ERR_PTR(-ENODEV); + } + return iommu_dev; } static void amd_iommu_release_device(struct device *dev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + if (dev_data->perfopt) { + if (WARN_ON(perfopt_put(iommu))) + dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n", + iommu->index); + dev_data->perfopt = false; + } WARN_ON(dev_data->domain); @@ -2925,6 +3121,19 @@ static int blocked_domain_attach_device(struct iommu_domain *domain, struct iommu_domain *old) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + /* + * blocked_domain is also the .release_domain, so this is the normal + * teardown path: drop the reference and clear the flag here too, and + * don't fail teardown if the WARN-guarded write doesn't stick. + */ + if (dev_data->perfopt) { + if (WARN_ON(perfopt_put(iommu))) + dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n", + iommu->index); + dev_data->perfopt = false; + } if (dev_data->domain) detach_device(dev); @@ -2993,6 +3202,9 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; + if (dev_data->perfopt && !pdom_is_in_pt_mode(domain)) + return -EBUSY; + /* * Skip attach device to domain if new domain is same as * devices current domain @@ -3137,12 +3349,8 @@ static void amd_iommu_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); } - region = iommu_alloc_resv_region(MSI_RANGE_START, - MSI_RANGE_END - MSI_RANGE_START + 1, - 0, IOMMU_RESV_MSI, GFP_KERNEL); - if (!region) + if (iommu_add_resv_x86_msi_region(head)) return; - list_add_tail(®ion->list, head); if (amd_iommu_ht_range_ignore()) return; @@ -3162,6 +3370,26 @@ static bool amd_iommu_is_attach_deferred(struct device *dev) return dev_data->defer_attach; } +static bool quirks_force_identity_mapping(struct pci_dev *pdev) +{ + int class = pdev->class >> 8; + + /* AMD GPU vendor ID */ + if (pdev->vendor != PCI_VENDOR_ID_ATI) + return false; + + /* GPU class */ + if (class != PCI_CLASS_DISPLAY_VGA && + class != PCI_CLASS_DISPLAY_OTHER) + return false; + + if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI) + return false; + + /* It is the GPU in an APU, force identity domain */ + return true; +} + static int amd_iommu_def_domain_type(struct device *dev) { struct iommu_dev_data *dev_data; @@ -3170,20 +3398,23 @@ static int amd_iommu_def_domain_type(struct device *dev) if (!dev_data) return 0; + if (!dev_is_pci(dev)) + return 0; + /* Always use DMA domain for untrusted device */ - if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) + if (to_pci_dev(dev)->untrusted) return IOMMU_DOMAIN_DMA; - /* - * Do not identity map IOMMUv2 capable devices when: - * - memory encryption is active, because some of those devices - * (AMD GPUs) don't have the encryption bit in their DMA-mask - * and require remapping. - * - SNP is enabled, because it prohibits DTE[Mode]=0. - */ - if (pdev_pasid_supported(dev_data) && - !cc_platform_has(CC_ATTR_MEM_ENCRYPT) && - !amd_iommu_snp_en) { + /* Apply device specific quirks */ + if (quirks_force_identity_mapping(to_pci_dev(dev))) { + /* + * When memory encryption is active, some of these devices + * don't have the encryption bit in their DMA-mask and + * require remapping. + */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + return 0; + return IOMMU_DOMAIN_IDENTITY; } @@ -3902,8 +4133,11 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, out_free_data: for (i--; i >= 0; i--) { irq_data = irq_domain_get_irq_data(domain, virq + i); - if (irq_data) - kfree(irq_data->chip_data); + if (irq_data && irq_data->chip_data) { + data = irq_data->chip_data; + kfree(data->entry); + kfree(data); + } } for (i = 0; i < nr_irqs; i++) free_irte(iommu, devid, index + i);
diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b160fb4..ebf4547 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c
@@ -21,6 +21,7 @@ #include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/iopoll.h> +#include <linux/minmax.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_address.h> @@ -267,6 +268,7 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + u64 mask; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -285,6 +287,13 @@ struct apple_dart_master_cfg { /* Intersection of DART capabilitles */ u32 supports_bypass : 1; + /* + * DMA aperture start and end to be overridden by "iommus"' phandle + * args. By default determined by DART's ias but may be outside of it. + */ + dma_addr_t dma_min; + dma_addr_t dma_max; + struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; }; @@ -537,7 +546,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova); + return ops->iova_to_phys(ops, iova & dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -551,8 +560,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, - mapped); + return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + pgcount, prot, gfp, mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -563,7 +572,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + gather); } static void @@ -590,6 +600,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; + u32 ias = min_t(u32, dart->ias, fls64(cfg->dma_max)); int ret = 0; int i, j; @@ -610,7 +621,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, - .ias = dart->ias, + .ias = ias, .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, @@ -623,10 +634,10 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } + dart_domain->mask = DMA_BIT_MASK(pgtbl_cfg.ias); dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - dart_domain->domain.geometry.aperture_start = 0; - dart_domain->domain.geometry.aperture_end = - (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); + dart_domain->domain.geometry.aperture_start = cfg->dma_min; + dart_domain->domain.geometry.aperture_end = cfg->dma_max; dart_domain->domain.geometry.force_aperture = true; dart_domain->finalized = true; @@ -803,20 +814,65 @@ static int apple_dart_of_xlate(struct device *dev, struct platform_device *iommu_pdev = of_find_device_by_node(args->np); struct apple_dart *dart = platform_get_drvdata(iommu_pdev); struct apple_dart *cfg_dart; + dma_addr_t dma_max = DMA_BIT_MASK(dart->ias); + dma_addr_t dma_min = 0; int i, sid; put_device(&iommu_pdev->dev); - if (args->args_count != 1) + if (args->args_count != 1 && args->args_count != 5) return -EINVAL; + sid = args->args[0]; + if (args->args_count == 5) { + dma_addr_t length = ((dma_addr_t)args->args[3] << 32) | args->args[4]; + + if (!length) + return -EINVAL; + + dma_min = ((dma_addr_t)args->args[1] << 32) | args->args[2]; + + if (!IS_ALIGNED(dma_min, dart->pgsize) || + !IS_ALIGNED(length, dart->pgsize)) { + dev_err(dev, "Unaligned DMA window %pad, %pad (0x%x)\n", + &dma_min, &length, dart->pgsize); + return -EINVAL; + } + if (check_add_overflow(dma_min, length - 1, &dma_max)) { + dev_err(dev, "DMA window length (%pad) overflows range for start %pad\n", + &length, &dma_min); + return -EINVAL; + } + + /* + * Ensure that the DMA window does not exceed the DART's ias. + */ + if ((dma_min ^ dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(dev, "Invalid DMA window for ias=%d\n", + dart->ias); + return -EINVAL; + } + } + if (!cfg) { cfg = kzalloc_obj(*cfg); if (!cfg) return -ENOMEM; /* Will be ANDed with DART capabilities */ cfg->supports_bypass = true; + /* Will be merged with other DARTs to the common range. */ + cfg->dma_min = dma_min; + cfg->dma_max = dma_max; + } else { + if (dma_min >= cfg->dma_max || cfg->dma_min >= dma_max) { + dev_err(dev, "non-overlapping DMA windows: %pad..%pad, %pad..%pad\n", + &dma_min, &dma_max, + &cfg->dma_min, &cfg->dma_max); + return -EINVAL; + } + cfg->dma_min = max(dma_min, cfg->dma_min); + cfg->dma_max = min(dma_max, cfg->dma_max); } dev_iommu_priv_set(dev, cfg);
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index ab1078a..e68b464 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
@@ -194,6 +194,7 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, arm_smmu_install_ste_for_dev(master, &ste); arm_smmu_attach_commit(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); return 0; }
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 0a429c6..faeec65 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -140,16 +140,19 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, { struct arm_smmu_domain *smmu_domain = container_of(mn, struct arm_smmu_domain, mmu_notifier); - size_t size; + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .iova = start, + /* + * The mm_types defines vm_end as the first byte after the end + * address, different from IOMMU subsystem using the last + * address of an address range. + */ + .size = end - start, + .iopte_size = PAGE_SIZE, + }; - /* - * The mm_types defines vm_end as the first byte after the end address, - * different from IOMMU subsystem using the last address of an address - * range. So do a simple translation here by calculating size correctly. - */ - size = end - start; - - arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false); + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -215,6 +218,13 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) if (system_supports_haft()) feat_mask |= ARM_SMMU_FEAT_HAFT; + /* + * The workaround for ARM_SMMU_OPT_FULL_CONT_RANGE_INV requires range + * invalidation support. + */ + if (smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV) + feat_mask |= ARM_SMMU_FEAT_RANGE_INV; + if ((smmu->features & feat_mask) != feat_mask) return false; @@ -338,6 +348,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, * ARM_SMMU_FEAT_RANGE_INV is present */ smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; + smmu_domain->tgsz_lg2 = PAGE_SHIFT; smmu_domain->stage = ARM_SMMU_DOMAIN_SVA; smmu_domain->smmu = smmu;
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index add6713..5a4e45e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
@@ -24,7 +24,9 @@ struct arm_smmu_test_writer { static struct arm_smmu_ste bypass_ste; static struct arm_smmu_ste abort_ste; static struct arm_smmu_device smmu = { - .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR + .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR | + ARM_SMMU_FEAT_RANGE_INV, + .options = ARM_SMMU_OPT_FULL_CONT_RANGE_INV, }; static struct mm_struct sva_mm = { .pgd = (void *)0xdaedbeefdeadbeefULL, @@ -125,8 +127,8 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, { __le64 used_bits[NUM_ENTRY_QWORDS] = {}; - arm_smmu_get_ste_used(ste, used_bits); - pr_debug("STE used bits: "); + writer->ops->get_used(ste, used_bits); + pr_debug("Entry used bits: "); print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits, sizeof(used_bits), false); } @@ -643,8 +645,10 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, const int *ids, const int *users, const int *ssids) { - KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs); + KUNIT_ASSERT_EQ(test, invs->num_invs, num_invs); KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); + KUNIT_EXPECT_TRUE(test, invs->has_range_inv); + KUNIT_EXPECT_TRUE(test, invs->has_full_cont_range_inv); while (num_invs--) { KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users), @@ -654,38 +658,43 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, } static struct arm_smmu_invs invs1 = { + .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, - { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, - { .type = INV_TYPE_ATS, .id = 3, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, + { .smmu = &smmu, .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 3, }, }, }; static struct arm_smmu_invs invs2 = { + .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ - { .type = INV_TYPE_ATS, .id = 4, }, - { .type = INV_TYPE_ATS, .id = 5, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 4, }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, }, }; static struct arm_smmu_invs invs3 = { + .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ - { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ - { .type = INV_TYPE_ATS, .id = 6, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 6, }, }, }; static struct arm_smmu_invs invs4 = { + .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, - { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, - { .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, }; static struct arm_smmu_invs invs5 = { + .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, - { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ - { .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, }; static void arm_smmu_v3_invs_test(struct kunit *test) @@ -704,17 +713,22 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* New array */ test_a = arm_smmu_invs_alloc(0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + KUNIT_EXPECT_FALSE(test, test_a->has_range_inv); + KUNIT_EXPECT_FALSE(test, test_a->has_full_cont_range_inv); /* Test1: merge invs1 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs1); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]), 0, results1[0], results1[1], results1[2]); /* Test2: merge invs2 (new array) */ test_a = arm_smmu_invs_merge(test_b, &invs2); kfree(test_b); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]), 0, results2[0], results2[1], results2[2]); @@ -726,6 +740,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test4: merge invs3 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs3); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]), 0, results4[0], results4[1], results4[2]); @@ -737,6 +752,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test6: purge test_b (new array) */ test_a = arm_smmu_invs_purge(test_b); kfree(test_b); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results6[0]), 0, results6[0], results6[1], results6[2]); @@ -748,12 +764,14 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test8: merge invs4 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs4); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results7[0]), 0, results7[0], results7[1], results7[2]); /* Test9: merge invs5 (new array) */ test_a = arm_smmu_invs_merge(test_b, &invs5); kfree(test_b); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results8[0]), 0, results8[0], results8[1], results8[2]); @@ -765,6 +783,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test11: purge test_a (new array) */ test_b = arm_smmu_invs_purge(test_a); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results10[0]), 0, results10[0], results10[1], results10[2]);
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 98f1057..6d4520c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -40,6 +40,11 @@ module_param(disable_msipolling, bool, 0444); MODULE_PARM_DESC(disable_msipolling, "Disable MSI-based polling for CMD_SYNC completion."); +static unsigned int cmdq_max_n_shift = CMDQ_MAX_SZ_SHIFT; +module_param(cmdq_max_n_shift, uint, 0444); +MODULE_PARM_DESC(cmdq_max_n_shift, + "Cap on the command queue depth, as log2 of the number of entries. Defaults to the hardware maximum; the queue never shrinks below one page. A kdump kernel always uses one page."); + static const struct iommu_ops arm_smmu_ops; static struct iommu_dirty_ops arm_smmu_dirty_ops; @@ -91,6 +96,7 @@ DEFINE_MUTEX(arm_smmu_asid_lock); static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"}, + { ARM_SMMU_OPT_OVR_INSTCFG_DATA, "arm,instdata-override"}, { 0, NULL}, }; @@ -181,8 +187,8 @@ static void queue_sync_cons_out(struct arm_smmu_queue *q) static void queue_inc_cons(struct arm_smmu_ll_queue *q) { - u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; - q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); + u32 cons = Q_POS(q, q->cons) + 1; + q->cons = Q_OVF(q->cons) | Q_POS(q, cons); } static void queue_sync_cons_ovf(struct arm_smmu_queue *q) @@ -192,8 +198,7 @@ static void queue_sync_cons_ovf(struct arm_smmu_queue *q) if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons))) return; - llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | - Q_IDX(llq, llq->cons); + llq->cons = Q_OVF(llq->prod) | Q_POS(llq, llq->cons); queue_sync_cons_out(q); } @@ -218,8 +223,8 @@ static int queue_sync_prod_in(struct arm_smmu_queue *q) static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n) { - u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; - return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); + u32 prod = Q_POS(q, q->prod) + n; + return Q_OVF(q->prod) | Q_POS(q, prod); } static void queue_poll_init(struct arm_smmu_device *smmu, @@ -1053,6 +1058,19 @@ static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l, return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]); } +static void arm_smmu_invs_update_caps(struct arm_smmu_invs *invs, + const struct arm_smmu_inv *inv) +{ + if (arm_smmu_inv_is_ats(inv)) + invs->has_ats = true; + + if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + invs->has_range_inv = true; + if (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV) + invs->has_full_cont_range_inv = true; + } +} + /** * arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for * arm_smmu_invs_merge() or arm_smmu_invs_unref() @@ -1123,8 +1141,7 @@ struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, */ if (new != new_invs->inv) WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1); - if (arm_smmu_inv_is_ats(new)) - new_invs->has_ats = true; + arm_smmu_invs_update_caps(new_invs, new); new++; } @@ -1234,8 +1251,7 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) arm_smmu_invs_for_each_entry(invs, i, inv) { new_invs->inv[num_invs] = *inv; - if (arm_smmu_inv_is_ats(inv)) - new_invs->has_ats = true; + arm_smmu_invs_update_caps(new_invs, inv); num_invs++; } @@ -1271,7 +1287,8 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR | STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH | STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | - STRTAB_STE_1_EATS | STRTAB_STE_1_MEV); + STRTAB_STE_1_EATS | STRTAB_STE_1_MEV | + STRTAB_STE_1_INSTCFG); used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); /* @@ -1287,7 +1304,8 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) if (cfg & BIT(1)) { used_bits[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS | - STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV); + STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV | + STRTAB_STE_1_INSTCFG); used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR | STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI | @@ -1515,6 +1533,9 @@ static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst, { u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V; + /* Ensure the zero-cleared L2 table is fully visible. */ + dma_wmb(); + /* The HW has 64 bit atomicity with stores to the L2 CD table */ WRITE_ONCE(dst->l2ptr, cpu_to_le64(val)); } @@ -1804,6 +1825,9 @@ static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst, val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1); val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; + /* Ensure the new L2 table is fully visible. */ + dma_wmb(); + /* The HW has 64 bit atomicity with stores to the L2 STE table */ WRITE_ONCE(dst->l2ptr, cpu_to_le64(val)); } @@ -1900,7 +1924,11 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, STRTAB_STE_1_S1STALLD : 0) | FIELD_PREP(STRTAB_STE_1_EATS, - ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); + ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) | + FIELD_PREP(STRTAB_STE_1_INSTCFG, + smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA ? + STRTAB_STE_1_INSTCFG_DATA : + STRTAB_STE_1_INSTCFG_INCOMING)); if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) && s1dss == STRTAB_STE_1_S1DSS_BYPASS) @@ -1952,7 +1980,11 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, target->data[1] = cpu_to_le64( FIELD_PREP(STRTAB_STE_1_EATS, - ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); + ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) | + FIELD_PREP(STRTAB_STE_1_INSTCFG, + smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA ? + STRTAB_STE_1_INSTCFG_DATA : + STRTAB_STE_1_INSTCFG_INCOMING)); if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB) target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB); @@ -2363,8 +2395,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) return IRQ_WAKE_THREAD; } -static struct arm_smmu_cmd -arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) +static struct arm_smmu_cmd arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, + struct arm_smmu_tlbi *tlbi) { size_t log2_span; size_t span_mask; @@ -2386,8 +2418,8 @@ arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) * This has the unpleasant side-effect of invalidating all PASID-tagged * ATC entries within the address range. */ - page_start = iova >> inval_grain_shift; - page_end = (iova + size - 1) >> inval_grain_shift; + page_start = tlbi->iova >> inval_grain_shift; + page_end = (tlbi->iova + tlbi->size - 1) >> inval_grain_shift; /* * In an ATS Invalidate Request, the address must be aligned on the @@ -2460,126 +2492,231 @@ static void arm_smmu_tlb_inv_context(void *cookie) arm_smmu_domain_inv(smmu_domain); } -static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, bool leaf, - unsigned long iova, size_t size, - size_t granule, size_t pgsize) +/* + * Check address alignment for TTL hint per SMMUv3 H.a Section 4.4.1. Address + * bits below the alignment must be zero, otherwise UNPREDICTABLE. + */ +static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg, + unsigned int ttl) { - unsigned long end = iova + size, num_pages = 0, tg = pgsize; - u64 orig_data0 = cmd->data[0]; - size_t inv_range = granule; - u8 ttl = 0, tg_enc = 0; + unsigned int pgsz_lg2 = arm_smmu_pt_level_to_lg2sz(tg, 3 - ttl); - if (WARN_ON_ONCE(!size)) - return; - - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - num_pages = size >> tg; - - /* Convert page size of 12,14,16 (log2) to 1,2,3 */ - tg_enc = (tg - 10) / 2; - - /* - * Determine what level the granule is at. For non-leaf, both - * io-pgtable and SVA pass a nominal last-level granule because - * they don't know what level(s) actually apply, so ignore that - * and leave TTL=0. However for various errata reasons we still - * want to use a range command, so avoid the SVA corner case - * where both scale and num could be 0 as well. - */ - if (leaf) - ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); - else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1) - num_pages++; - } - - while (iova < end) { - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - /* - * On each iteration of the loop, the range is 5 bits - * worth of the aligned size remaining. - * The range in pages is: - * - * range = (num_pages & (0x1f << __ffs(num_pages))) - */ - unsigned long scale, num; - - /* Determine the power of 2 multiple number of pages */ - scale = __ffs(num_pages); - - /* Determine how many chunks of 2^scale size we have */ - num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; - - /* Keep the pre-DS 5-bit truncation when scale > 31 */ - cmd->data[0] = orig_data0 | - FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) | - FIELD_PREP(CMDQ_TLBI_0_SCALE, scale & 0x1f); - - /* range is num * 2^scale * pgsize */ - inv_range = num << (scale + tg); - - /* Clear out the lower order bits for the next iteration */ - num_pages -= num << scale; - } - - /* - * IPA has fewer bits than VA, but they are reserved in the - * command and something would be very broken if iova had them - * set. - */ - cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) | - FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | - FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | - (iova & ~GENMASK_U64(11, 0)); - - arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); - iova += inv_range; - } + return !(address & GENMASK_U64(pgsz_lg2 - 1, 0)); } -static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, - size_t granule) +struct arm_smmu_range_inv { + u64 start_tg; + /* Normal integer, not encoded. 0 means 0.*/ + u64 num; + unsigned int scale; +}; + +static unsigned int arm_smmu_range_inv_calc_scale(u64 num_tg) { - size_t max_tlbi_ops; + return fls64((num_tg - 1) / (CMDQ_TLBI_RANGE_NUM_MAX + 1)); +} - /* 0 size means invalidate all */ - if (!size || size == SIZE_MAX) - return true; +static u64 arm_smmu_range_inv_calc_num(u64 num_tg, unsigned int scale) +{ + return DIV_ROUND_UP_ULL(num_tg, 1ULL << scale); +} - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) - return false; +/* + * Initialize the smallest range invalidation covering num_tg and ending at + * last_tg. + */ +static struct arm_smmu_range_inv arm_smmu_range_inv_init_end(u64 last_tg, + u64 num_tg) +{ + struct arm_smmu_range_inv range_inv = {}; + + if (!num_tg) + return range_inv; + + range_inv.scale = arm_smmu_range_inv_calc_scale(num_tg); + range_inv.num = arm_smmu_range_inv_calc_num(num_tg, range_inv.scale); + range_inv.start_tg = last_tg - ((range_inv.num << range_inv.scale) - 1); + return range_inv; +} + +static void +arm_smmu_tlbi_add_range_cmd(struct arm_smmu_tlbi *tlbi, + const struct arm_smmu_range_inv *range_inv, u8 ttl, + u8 tg_enc) +{ + struct arm_smmu_cmd *cmd = &tlbi->range.cmds[tlbi->range.num_cmds++]; + unsigned int tgsz_lg2 = tg_enc * 2 + 10; + u64 iova = range_inv->start_tg << tgsz_lg2; + unsigned int num = range_inv->num - 1; + + /* 16K granule TTL=1 is reserved (Section 4.4.1) */ + if (WARN_ON(tgsz_lg2 == 14 && ttl == 1)) + ttl = 0; + + /* Verify address alignment for the TTL hint */ + if (ttl && !arm_smmu_ttl_addr_aligned(iova, tgsz_lg2, ttl)) + ttl = 0; /* - * Borrowed from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, - * this is used as a threshold to replace "size_opcode" commands with a - * single "nsize_opcode" command, when SMMU doesn't implement the range - * invalidation feature, where there can be too many per-granule TLBIs, - * resulting in a soft lockup. + * SMMUv3 H.a Section 4.4.1: TG!=0, NUM==0, SCALE==0, TTL==0 is Reserved + * and causes CERROR_ILL. Single tg uses NUM=0, SCALE=0 with a TTL hint + * to target only the exact leaf entry. + * + * For a single tg invalidation a 0 TTL can come from places like the + * SVA path that don't have enough information to get a TTL, or as a + * side of effect of the splitting. + * + * A single-TG invalidation cannot reach this point if it is part of a + * CONT group, so it is safe to transform it into a single invalidation. + * The ARM_SMMU_OPT_FULL_CONT_RANGE_INV errata does not apply. */ - max_tlbi_ops = 1 << (ilog2(granule) - 3); - return size >= max_tlbi_ops * granule; + if (!num && !range_inv->scale && !ttl) + tg_enc = 0; + + cmd->data[0] = FIELD_PREP(CMDQ_TLBI_0_NUM, num) | + FIELD_PREP(CMDQ_TLBI_0_SCALE, range_inv->scale); + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | + FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | + FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | iova; } -/* Used by non INV_TYPE_ATS* invalidations */ -static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, - bool leaf, - unsigned long iova, size_t size, - unsigned int granule) +/* + * Generate up to two range TLBI command payloads covering [iova, iova+size). + * Sets use_full_inv if the range is too large to represent. + * + * Normally the first range invalidation is the largest representable span which + * does not exceed the requested range. If necessary, the second range + * invalidation is the smallest representable range covering the remainder and + * is anchored at the end. Any excess coverage from the second range + * invalidation overlaps the first instead of exceeding the requested range. + * + * For SVA on an invs containing an SMMU with ARM_SMMU_OPT_FULL_CONT_RANGE_INV, + * produce only a single range invalidation and overinvalidate so any potential CONT is + * covered by one command. + */ +static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi, + bool single_range_inv) { - if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) { - struct arm_smmu_cmd nsize_cmd = *cmd; + u8 tgsz_lg2 = tlbi->tgsz_lg2; + struct arm_smmu_range_inv first = { .start_tg = tlbi->iova >> + tgsz_lg2 }; + u64 last_tg = (tlbi->iova + tlbi->size - 1) >> tgsz_lg2; + u64 num_tg = last_tg - first.start_tg + 1; + u8 tg_enc = (tgsz_lg2 - 10) / 2; + struct arm_smmu_range_inv trail; + u8 ttl = 0; - u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode, - CMDQ_0_OP); - arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd); + /* + * Determine what level the granule is at. For non-leaf, both io-pgtable + * and SVA pass a nominal last-level granule because they don't know + * what level(s) actually apply, so leave TTL=0. + */ + if (tlbi->leaf_only) + ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tgsz_lg2 - 3)); + + /* + * The spec defines the invalidated range as: + * Range = ((NUM+1) * 2^SCALE) * Translation_Granule_Size + * NUM is 5 bits, so (NUM+1) covers 1..32 granules. Find the smallest + * SCALE at which a single command could cover num_tg. + * + * Unlike other IOMMUs the spec has no alignment requirement on the + * address beyond alignment to tg (so long as TTL=0). + */ + first.scale = arm_smmu_range_inv_calc_scale(num_tg); + if (first.scale > 31) { + /* Range too large for a single command do full invalidation */ + tlbi->range.use_full_inv = true; return; } - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, - iova, size, granule, inv->pgsize); + if (single_range_inv) { + /* + * Produce a single invalidation by rounding up and disabling + * the trailer. + */ + first.num = arm_smmu_range_inv_calc_num(num_tg, first.scale); + trail.num = 0; + } else { + /* + * Produce two invalidations by rounding down and adding a + * second trailing range invalidation anchored at the end. + */ + first.num = num_tg >> first.scale; + trail = arm_smmu_range_inv_init_end( + last_tg, num_tg - ((u64)first.num << first.scale)); + } + arm_smmu_tlbi_add_range_cmd(tlbi, &first, ttl, tg_enc); + + if (trail.num) + arm_smmu_tlbi_add_range_cmd(tlbi, &trail, ttl, tg_enc); +} + +/* + * One TLBI command per IOTLB entry, assuming the entries are all at least + * iopte_granule sized. Sets use_full_inv if too many commands would be needed + * which indicates too high a latency. The threshold is similar to MAX_DVM_OPS + * in arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE. + */ +static void arm_smmu_tlbi_calc_single(struct arm_smmu_tlbi *tlbi) +{ + unsigned long num_ops = tlbi->size / tlbi->iopte_size; + + if (!num_ops || num_ops > 512) { + tlbi->single.use_full_inv = true; + return; + } + tlbi->single.num = num_ops; +} + +static void arm_smmu_inv_all_cmd(struct arm_smmu_inv *inv, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd) +{ + u64p_replace_bits(&cmd->data[0], inv->nsize_opcode, CMDQ_0_OP); + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd); +} + +/* + * Used by non INV_TYPE_ATS* invalidations. Returns true if it fell back to + * full invalidation using nsize_opcode. + */ +static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd, + struct arm_smmu_tlbi *tlbi) +{ + u64 iova = tlbi->iova; + unsigned int i; + + if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + if (tlbi->range.use_full_inv) { + arm_smmu_inv_all_cmd(inv, cmds, cmd); + return true; + } + for (i = 0; i < tlbi->range.num_cmds; i++) { + struct arm_smmu_cmd range_cmd = tlbi->range.cmds[i]; + + range_cmd.data[0] |= cmd->data[0]; + range_cmd.data[1] |= cmd->data[1]; + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, + &range_cmd); + } + return false; + } + + if (tlbi->single.use_full_inv) { + arm_smmu_inv_all_cmd(inv, cmds, cmd); + return true; + } + + for (i = 0; i < tlbi->single.num; i++) { + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | + (iova & ~GENMASK_U64(11, 0)); + iova += tlbi->iopte_size; + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd); + } + return false; } static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, @@ -2598,10 +2735,10 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, return false; } -static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, - unsigned long iova, size_t size, - unsigned int granule, bool leaf) +static void arm_smmu_domain_tlbi_inv(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_invs *invs) { + struct arm_smmu_inv *used_s12_vmall = NULL; struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_inv *cur; struct arm_smmu_inv *end; @@ -2630,18 +2767,24 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, case INV_TYPE_S1_ASID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, cur->id, 0); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf, - iova, size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); break; case INV_TYPE_S2_VMID: - cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, - 0, cur->id); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf, - iova, size, granule); + cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, 0, + cur->id); + if (arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi)) + used_s12_vmall = cur + 1; break; case INV_TYPE_S2_VMID_S1_CLEAR: - /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ - if (arm_smmu_inv_size_too_big(cur->smmu, size, granule)) + /* + * S2_VMID used CMDQ_OP_TLBI_S12_VMALL which already + * flushed S1 entries. These two types always come in + * pairs and arm_smmu_inv_cmp() ensures that they are + * consecutive in the list for the same SMMU. There may + * be several pairings so check this is paired with the + * one that did the full invalidation. + */ + if (used_s12_vmall == cur) break; arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, @@ -2652,7 +2795,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid, - iova, size)); + tlbi)); break; case INV_TYPE_ATS_FULL: arm_smmu_cmdq_batch_add_cmd( @@ -2679,12 +2822,20 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, } } -void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, - unsigned long iova, size_t size, - unsigned int granule, bool leaf) +/* + * Perform the invalidation with the parameters described by tlbi on everything + * caching for smmu_domain. tlbi is an on-stack structure that contains both the + * input description and working memory for this function it must be discarded + * after this returns. + */ +void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_domain *smmu_domain) { struct arm_smmu_invs *invs; + if (!tlbi->single.use_full_inv) + arm_smmu_tlbi_calc_single(tlbi); + /* * An invalidation request must follow some IOPTE change and then load * an invalidation array. In the meantime, a domain attachment mutates @@ -2702,7 +2853,7 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, * * [CPU0] | [CPU1] * change IOPTE on new domain: | - * arm_smmu_domain_inv_range() { | arm_smmu_install_new_domain_invs() + * arm_smmu_domain_tlbi() { | arm_smmu_install_new_domain_invs() * smp_mb(); // ensures IOPTE | arm_smmu_install_ste_for_dev { * // seen by SMMU | dma_wmb(); // ensures invs update * // load the updated invs | // before updating STE @@ -2716,6 +2867,20 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, invs = rcu_dereference(smmu_domain->invs); /* + * Only precompute range invalidation commands when they will be used. + * The invs generation ensures this matches the instances being + * invalidated. + */ + if (invs->has_range_inv) { + if (!tlbi->range.use_full_inv) { + arm_smmu_tlbi_calc_range( + tlbi, + smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && + invs->has_full_cont_range_inv); + } + } + + /* * Avoid locking unless ATS is being used. No ATC invalidation can be * going on after a domain is detached. */ @@ -2723,10 +2888,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long flags; read_lock_irqsave(&invs->rwlock, flags); - __arm_smmu_domain_inv_range(invs, iova, size, granule, leaf); + arm_smmu_domain_tlbi_inv(tlbi, invs); read_unlock_irqrestore(&invs->rwlock, flags); } else { - __arm_smmu_domain_inv_range(invs, iova, size, granule, leaf); + arm_smmu_domain_tlbi_inv(tlbi, invs); } rcu_read_unlock(); @@ -2742,12 +2907,23 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, iommu_iotlb_gather_add_page(domain, gather, iova, granule); } +/* + * Called by io-pgtable-arm.c for each single table level it wants to remove. + * size is the size of the table level and granule is the tg in bytes. This must + * clear the walk cache and any leaves within the range. + */ static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size, size_t granule, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .iova = iova, + .size = size, + .iopte_size = 1 << smmu_domain->tgsz_lg2, + }; - arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false); + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } static const struct iommu_flush_ops arm_smmu_flush_ops = { @@ -2933,6 +3109,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, return -ENOMEM; smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + smmu_domain->tgsz_lg2 = __ffs(pgtbl_cfg.pgsize_bitmap); smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; smmu_domain->domain.geometry.force_aperture = true; if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1) @@ -3172,15 +3349,13 @@ static void arm_smmu_disable_iopf(struct arm_smmu_master *master, static struct arm_smmu_inv * arm_smmu_master_build_inv(struct arm_smmu_master *master, - enum arm_smmu_inv_type type, u32 id, ioasid_t ssid, - size_t pgsize) + enum arm_smmu_inv_type type, u32 id, ioasid_t ssid) { struct arm_smmu_invs *build_invs = master->build_invs; struct arm_smmu_inv *cur, inv = { .smmu = master->smmu, .type = type, .id = id, - .pgsize = pgsize, }; if (WARN_ON(build_invs->num_invs >= build_invs->max_invs)) @@ -3232,28 +3407,24 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, ioasid_t ssid, struct arm_smmu_domain *smmu_domain) { const bool nesting = smmu_domain->nest_parent; - size_t pgsize = 0, i; + size_t i; iommu_group_mutex_assert(master->dev); master->build_invs->num_invs = 0; - /* Range-based invalidation requires the leaf pgsize for calculation */ - if (master->smmu->features & ARM_SMMU_FEAT_RANGE_INV) - pgsize = __ffs(smmu_domain->domain.pgsize_bitmap); - switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_SVA: case ARM_SMMU_DOMAIN_S1: if (!arm_smmu_master_build_inv(master, INV_TYPE_S1_ASID, smmu_domain->cd.asid, - IOMMU_NO_PASID, pgsize)) + IOMMU_NO_PASID)) return NULL; break; case ARM_SMMU_DOMAIN_S2: if (!arm_smmu_master_build_inv(master, INV_TYPE_S2_VMID, smmu_domain->s2_cfg.vmid, - IOMMU_NO_PASID, pgsize)) + IOMMU_NO_PASID)) return NULL; break; default: @@ -3265,7 +3436,7 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, if (nesting) { if (!arm_smmu_master_build_inv( master, INV_TYPE_S2_VMID_S1_CLEAR, - smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID, 0)) + smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID)) return NULL; } @@ -3276,7 +3447,7 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, */ if (!arm_smmu_master_build_inv( master, nesting ? INV_TYPE_ATS_FULL : INV_TYPE_ATS, - master->streams[i].id, ssid, 0)) + master->streams[i].id, ssid)) return NULL; } @@ -3285,9 +3456,9 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, return master->build_invs; } -static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, - struct iommu_domain *domain, - ioasid_t ssid) +static struct arm_smmu_master_domain * +arm_smmu_remove_master_domain(struct arm_smmu_master *master, + struct iommu_domain *domain, ioasid_t ssid) { struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain); struct arm_smmu_master_domain *master_domain; @@ -3295,7 +3466,7 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, unsigned long flags; if (!smmu_domain) - return; + return NULL; if (domain->type == IOMMU_DOMAIN_NESTED) nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats; @@ -3310,8 +3481,24 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + /* arm_smmu_attach_release() will free it */ + return master_domain; +} + +/* Release the old master_domain detached by arm_smmu_remove_master_domain() */ +void arm_smmu_attach_release(struct arm_smmu_attach_state *state) +{ + struct arm_smmu_master_domain *master_domain = state->old_master_domain; + struct arm_smmu_master *master = state->master; + + iommu_group_mutex_assert(master->dev); + + if (!master_domain) + return; + arm_smmu_disable_iopf(master, master_domain); kfree(master_domain); + state->old_master_domain = NULL; } /* @@ -3609,7 +3796,8 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state) arm_smmu_atc_inv_master(master, IOMMU_NO_PASID); } - arm_smmu_remove_master_domain(master, state->old_domain, state->ssid); + state->old_master_domain = arm_smmu_remove_master_domain( + master, state->old_domain, state->ssid); arm_smmu_install_old_domain_invs(state); master->ats_enabled = state->ats_enabled; } @@ -3684,6 +3872,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev, arm_smmu_attach_commit(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); return 0; } @@ -3766,8 +3955,10 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, mutex_lock(&arm_smmu_asid_lock); ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain); - if (ret) - goto out_unlock; + if (ret) { + mutex_unlock(&arm_smmu_asid_lock); + return ret; + } /* * We don't want to obtain to the asid_lock too early, so fix up the @@ -3781,10 +3972,9 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, arm_smmu_update_ste(master, sid_domain, state.ats_enabled); arm_smmu_attach_commit(&state); - -out_unlock: mutex_unlock(&arm_smmu_asid_lock); - return ret; + arm_smmu_attach_release(&state); + return 0; } static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, @@ -3804,9 +3994,11 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, arm_smmu_clear_cd(master, pasid); if (master->ats_enabled) arm_smmu_atc_inv_master(master, pasid); - arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid); + state.old_master_domain = arm_smmu_remove_master_domain( + master, &smmu_domain->domain, pasid); arm_smmu_install_old_domain_invs(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); /* * When the last user of the CD table goes away downgrade the STE back @@ -3869,6 +4061,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, arm_smmu_install_ste_for_dev(master, ste); arm_smmu_attach_commit(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); /* * This has to be done after removing the master from the @@ -4025,13 +4218,18 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .iova = gather->start, + .size = gather->end - gather->start + 1, + .iopte_size = gather->pgsize, + .leaf_only = true, + }; if (!gather->pgsize) return; - arm_smmu_domain_inv_range(smmu_domain, gather->start, - gather->end - gather->start + 1, - gather->pgsize, true); + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } static phys_addr_t @@ -4122,6 +4320,13 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, sizeof(master->streams[0]), arm_smmu_stream_id_cmp, NULL); + /* + * Clear after sorting: RB_CLEAR_NODE() records the node's own address, + * which sort_nonatomic() invalidates by relocating the entries. + */ + for (i = 0; i < fwspec->num_ids; i++) + RB_CLEAR_NODE(&master->streams[i].node); + mutex_lock(&smmu->streams_mutex); for (i = 0; i < fwspec->num_ids; i++) { struct arm_smmu_stream *new_stream = &master->streams[i]; @@ -4154,7 +4359,9 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, if (ret) { for (i--; i >= 0; i--) - rb_erase(&master->streams[i].node, &smmu->streams); + if (!RB_EMPTY_NODE(&master->streams[i].node)) + rb_erase(&master->streams[i].node, + &smmu->streams); kfree(master->streams); kfree(master->build_invs); } @@ -4174,7 +4381,8 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) mutex_lock(&smmu->streams_mutex); for (i = 0; i < fwspec->num_ids; i++) - rb_erase(&master->streams[i].node, &smmu->streams); + if (!RB_EMPTY_NODE(&master->streams[i].node)) + rb_erase(&master->streams[i].node, &smmu->streams); mutex_unlock(&smmu->streams_mutex); kfree(master->streams); @@ -4412,6 +4620,56 @@ static struct iommu_dirty_ops arm_smmu_dirty_ops = { }; /* Probing and initialisation functions */ + +/** + * arm_smmu_queue_max_n_shift() - pick the log2 depth of a queue + * @hw_max_n_shift: log2 depth the hardware advertises in IDR1 + * @ent_sz_shift: log2 of the queue entry size in bytes + * @limit_n_shift: log2 depth to cap the queue at + * + * A kdump capture kernel gets one page worth of entries whatever the limit. + * + * @limit_n_shift is floored at one page, because coherent DMA is page + * granular: a shallower queue occupies the same memory as one that fills the + * page, and arm_smmu_init_one_queue() stops shrinking at a page too. + */ +static u32 arm_smmu_queue_max_n_shift(u32 hw_max_n_shift, u32 ent_sz_shift, + u32 limit_n_shift) +{ + u32 floor_n_shift = PAGE_SHIFT - ent_sz_shift; + + if (is_kdump_kernel()) + return min(hw_max_n_shift, floor_n_shift); + + limit_n_shift = max(limit_n_shift, floor_n_shift); + return min(hw_max_n_shift, limit_n_shift); +} + +static inline u32 arm_smmu_evtq_max_n_shift(u32 hw_max_n_shift) +{ + /* Capped to ensure natural alignment */ + return arm_smmu_queue_max_n_shift(hw_max_n_shift, EVTQ_ENT_SZ_SHIFT, + EVTQ_MAX_SZ_SHIFT); +} + +static inline u32 arm_smmu_priq_max_n_shift(u32 hw_max_n_shift) +{ + /* Capped to ensure natural alignment */ + return arm_smmu_queue_max_n_shift(hw_max_n_shift, PRIQ_ENT_SZ_SHIFT, + PRIQ_MAX_SZ_SHIFT); +} + +/* + * Command queues are also allocated by the Tegra241 CMDQV for its VCMDQs, which + * need the same depth decision. + */ +u32 arm_smmu_cmdq_max_n_shift(u32 hw_max_n_shift) +{ + /* Capped to ensure natural alignment */ + return arm_smmu_queue_max_n_shift(hw_max_n_shift, CMDQ_ENT_SZ_SHIFT, + min(CMDQ_MAX_SZ_SHIFT, cmdq_max_n_shift)); +} + int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, struct arm_smmu_queue *q, void __iomem *page, unsigned long prod_off, unsigned long cons_off, @@ -5009,12 +5267,21 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) /* Arm errata 2268618, 2812531 */ smmu->features &= ~ARM_SMMU_FEAT_NESTING; } + /* Arm errata 3777127 */ + smmu->options |= ARM_SMMU_OPT_FULL_CONT_RANGE_INV; break; case IIDR_PRODUCTID_ARM_MMU_L1: - case IIDR_PRODUCTID_ARM_MMU_S3: - /* Arm errata 3878312/3995052 */ + /* Arm errata 3878312 */ smmu->features &= ~ARM_SMMU_FEAT_BTM; break; + case IIDR_PRODUCTID_ARM_MMU_S3: + /* Arm errata 3995052 */ + smmu->features &= ~ARM_SMMU_FEAT_BTM; + /* Arm errata 3673557 */ + if (variant < 1 || (variant == 1 && revision < 1)) + smmu->options |= + ARM_SMMU_OPT_FULL_CONT_RANGE_INV; + break; } break; } @@ -5155,9 +5422,14 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (reg & IDR1_ATTR_TYPES_OVR) smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR; - /* Queue sizes, capped to ensure natural alignment */ - smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, - FIELD_GET(IDR1_CMDQS, reg)); + if ((smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA) && + !(reg & IDR1_ATTR_PERMS_OVR)) { + dev_err(smmu->dev, "Inst/Data attribute override not supported\n"); + return -ENXIO; + } + + smmu->cmdq.q.llq.max_n_shift = + arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, reg)); if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { /* * We don't support splitting up batches, so one batch of @@ -5170,10 +5442,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) return -ENXIO; } - smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, - FIELD_GET(IDR1_EVTQS, reg)); - smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, - FIELD_GET(IDR1_PRIQS, reg)); + smmu->evtq.q.llq.max_n_shift = + arm_smmu_evtq_max_n_shift(FIELD_GET(IDR1_EVTQS, reg)); + smmu->priq.q.llq.max_n_shift = + arm_smmu_priq_max_n_shift(FIELD_GET(IDR1_PRIQS, reg)); /* SID/SSID sizes */ smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); @@ -5633,7 +5905,7 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); - arm_smmu_device_disable(smmu); + arm_smmu_disable_action(smmu); } static const struct of_device_id arm_smmu_of_match[] = {
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index dd2fee2..bb048e0 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -52,6 +52,7 @@ struct arm_vsmmu; #define IDR1_QUEUES_PRESET (1 << 29) #define IDR1_REL (1 << 28) #define IDR1_ATTR_TYPES_OVR (1 << 27) +#define IDR1_ATTR_PERMS_OVR (1 << 26) #define IDR1_CMDQS GENMASK(25, 21) #define IDR1_EVTQS GENMASK(20, 16) #define IDR1_PRIQS GENMASK(15, 11) @@ -187,6 +188,8 @@ struct arm_vsmmu; #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1)) #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift)) +/* A position is Q_WRP | Q_IDX, wrapping at twice the queue capacity */ +#define Q_POS(llq, p) (Q_WRP(llq, p) | Q_IDX(llq, p)) #define Q_OVERFLOW_FLAG (1U << 31) #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG) #define Q_ENT(q, p) ((q)->base + \ @@ -285,6 +288,10 @@ static inline u32 arm_smmu_strtab_l2_idx(u32 sid) #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44) #define STRTAB_STE_1_SHCFG_INCOMING 1UL +#define STRTAB_STE_1_INSTCFG GENMASK_ULL(51, 50) +#define STRTAB_STE_1_INSTCFG_INCOMING 0UL +#define STRTAB_STE_1_INSTCFG_DATA 2UL + #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0) @@ -736,10 +743,9 @@ struct arm_smmu_inv { u8 size_opcode; u8 nsize_opcode; u32 id; /* ASID or VMID or SID */ - union { - size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */ - u32 ssid; /* INV_TYPE_ATS */ - }; + + /* Only used by INV_TYPE_ATS */ + u32 ssid; int users; /* users=0 to mark as a trash to be purged */ }; @@ -759,6 +765,9 @@ static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) * Must not be greater than @num_invs * @rwlock: optional rwlock to fence ATS operations * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL + * @has_range_inv: flag if any entry's SMMU supports range invalidation + * @has_full_cont_range_inv: flag if any entry's SMMU requires the CONT range + * invalidation workaround * @rcu: rcu head for kfree_rcu() * @inv: flexible invalidation array * @@ -788,6 +797,8 @@ struct arm_smmu_invs { size_t num_trashes; rwlock_t rwlock; bool has_ats; + bool has_range_inv; + bool has_full_cont_range_inv; struct rcu_head rcu; struct arm_smmu_inv inv[] __counted_by(max_invs); }; @@ -805,6 +816,40 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs) return new_invs; } +/* Generic page-table level 0 is the leaf-only level. */ +static inline unsigned int arm_smmu_pt_level_to_lg2sz(unsigned int tgsz_lg2, + unsigned int level) +{ + return tgsz_lg2 + (tgsz_lg2 - ilog2(sizeof(u64))) * level; +} + +static inline unsigned int arm_smmu_pt_lg2sz_to_level(unsigned int tgsz_lg2, + unsigned int lg2sz) +{ + return (lg2sz - tgsz_lg2) / (tgsz_lg2 - ilog2(sizeof(u64))); +} + +struct arm_smmu_tlbi { + unsigned long iova; + size_t size; + /* page or block size of the leaf iopte */ + unsigned int iopte_size; + /* Base Translation Granule of the page table */ + u8 tgsz_lg2; + bool leaf_only; + + struct { + bool use_full_inv; + u16 num; + } single; + + struct { + bool use_full_inv; + u8 num_cmds; + struct arm_smmu_cmd cmds[2]; + } range; +}; + struct arm_smmu_evtq { struct arm_smmu_queue q; struct iopf_queue *iopf; @@ -934,6 +979,12 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_MSIPOLL (1 << 2) #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4) +#define ARM_SMMU_OPT_OVR_INSTCFG_DATA (1 << 5) +/* + * Range invalidation is mandatory and one range invalidation must fully span an + * invalidated CONT + */ +#define ARM_SMMU_OPT_FULL_CONT_RANGE_INV (1 << 6) u32 options; struct arm_smmu_cmdq cmdq; @@ -1049,6 +1100,7 @@ struct arm_smmu_domain { spinlock_t devices_lock; bool enforce_cache_coherency : 1; bool nest_parent : 1; + u8 tgsz_lg2; struct mmu_notifier mmu_notifier; }; @@ -1154,17 +1206,24 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, struct arm_smmu_domain *smmu_domain, ioasid_t pasid, struct arm_smmu_cd *cd, struct iommu_domain *old); -void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, - unsigned long iova, size_t size, - unsigned int granule, bool leaf); +void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_domain *smmu_domain); static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain) { - arm_smmu_domain_inv_range(smmu_domain, 0, 0, 0, false); + /* Prefilled for invalidate all */ + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .single.use_full_inv = true, + .range.use_full_inv = true, + }; + + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, struct arm_smmu_cmdq *cmdq); +u32 arm_smmu_cmdq_max_n_shift(u32 hw_max_n_shift); int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, struct arm_smmu_queue *q, void __iomem *page, unsigned long prod_off, unsigned long cons_off, @@ -1204,12 +1263,14 @@ struct arm_smmu_attach_state { struct arm_smmu_vmaster *vmaster; struct arm_smmu_inv_state old_domain_invst; struct arm_smmu_inv_state new_domain_invst; + struct arm_smmu_master_domain *old_master_domain; bool ats_enabled; }; int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, struct iommu_domain *new_domain); void arm_smmu_attach_commit(struct arm_smmu_attach_state *state); +void arm_smmu_attach_release(struct arm_smmu_attach_state *state); void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master, const struct arm_smmu_ste *target);
diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 6644075..710a4c6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
@@ -663,7 +663,7 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq) /* Cap queue size to SMMU's IDR1.CMDQS and ensure natural alignment */ regval = readl_relaxed(smmu->base + ARM_SMMU_IDR1); q->llq.max_n_shift = - min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, regval)); + arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, regval)); /* Use the common helper to init the VCMDQ, and then... */ ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0,
diff --git a/drivers/iommu/bcm2712-iommu-cache.c b/drivers/iommu/bcm2712-iommu-cache.c new file mode 100644 index 0000000..8ed4369 --- /dev/null +++ b/drivers/iommu/bcm2712-iommu-cache.c
@@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * IOMMU driver for BCM2712 TLB cache + * + * Copyright (c) 2023 Raspberry Pi Ltd. + * Copyright (c) 2026 Daniel Drake + * + * The BCM2712 IOMMUC is a centralized TLB which accelerates address translation + * across the SoC's IOMMU devices. If an address mapping is not found in the + * IOMMU's local TLB cache, then this IOMMUC is consulted. The IOMMUC must be + * explicitly invalidated when modifying or unmapping IOMMU page tables. + */ + +#include <linux/bits.h> +#include <linux/cleanup.h> +#include <linux/err.h> +#include <linux/iopoll.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> + +#include "bcm2712-iommu-cache.h" + +struct bcm2712_iommu_cache { + struct device *dev; + spinlock_t hw_lock; + void __iomem *reg_base; +}; + +#define MMUC_CONTROL_ENABLE BIT(0) +#define MMUC_CONTROL_FLUSH BIT(1) +#define MMUC_CONTROL_FLUSHING BIT(2) + +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache) +{ + u32 val; + int ret; + + scoped_guard(spinlock_irqsave, &cache->hw_lock) { + writel(MMUC_CONTROL_ENABLE | MMUC_CONTROL_FLUSH, + cache->reg_base); + + ret = readl_poll_timeout_atomic(cache->reg_base, val, + !(val & MMUC_CONTROL_FLUSHING), + 0, 50); + } + + if (ret) + dev_err_ratelimited(cache->dev, "cache flush timed out\n"); +} + +static int bcm2712_iommu_cache_probe(struct platform_device *pdev) +{ + struct bcm2712_iommu_cache *cache; + + cache = devm_kzalloc(&pdev->dev, sizeof(*cache), GFP_KERNEL); + if (!cache) + return -ENOMEM; + + cache->dev = &pdev->dev; + spin_lock_init(&cache->hw_lock); + + cache->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(cache->reg_base)) + return PTR_ERR(cache->reg_base); + + platform_set_drvdata(pdev, cache); + return 0; +} + +static const struct of_device_id bcm2712_iommu_cache_of_match[] = { + { .compatible = "brcm,bcm2712-iommuc" }, + { /* sentinel */ }, +}; + +static struct platform_driver bcm2712_iommu_cache_driver = { + .probe = bcm2712_iommu_cache_probe, + .driver = { + .name = "bcm2712-iommu-cache", + .of_match_table = bcm2712_iommu_cache_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(bcm2712_iommu_cache_driver);
diff --git a/drivers/iommu/bcm2712-iommu-cache.h b/drivers/iommu/bcm2712-iommu-cache.h new file mode 100644 index 0000000..d2f7851 --- /dev/null +++ b/drivers/iommu/bcm2712-iommu-cache.h
@@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _BCM2712_IOMMU_CACHE_H +#define _BCM2712_IOMMU_CACHE_H + +struct bcm2712_iommu_cache; + +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache); + +#endif
diff --git a/drivers/iommu/bcm2712-iommu.c b/drivers/iommu/bcm2712-iommu.c new file mode 100644 index 0000000..3018554 --- /dev/null +++ b/drivers/iommu/bcm2712-iommu.c
@@ -0,0 +1,550 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * IOMMU driver for Broadcom BCM2712 + * + * Copyright (c) 2023-2025 Raspberry Pi Ltd. + * Copyright (c) 2026 Daniel Drake + * + * Each BCM2712 IOMMU has multiple devices hardwired into it, whose + * DMA transactions all route through the IOMMU. There is no stream ID tagging + * or any other kind of segmentation to differentiate between requests from + * different devices. It is also not possible to toggle a specific device + * between iommu-mapped and bypass modes. + * + * The hardware supports the simultaneous configuration of a bypass window + * (where memory accesses operate in identity mode) and a translation aperture + * (supporting page mapping). While combining these two would allow for + * GART-like operation, this driver's paging domain implementation configures + * only the translation aperture (starting at IOVA 0) offering a degree of + * memory protection. + * + * The page table format is a two-level format handled by generic_pt/bcm2712. + */ + +#include <linux/bitfield.h> +#include <linux/cleanup.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/iommu.h> +#include <linux/iopoll.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <linux/sizes.h> +#include <linux/generic_pt/iommu.h> + +#include "bcm2712-iommu-cache.h" +#include "dma-iommu.h" +#include "iommu-pages.h" + +/* BCM2712 IOMMU is organized around 4Kbyte pages */ +#define IOMMU_PAGE_SHIFT 12 +#define IOMMU_PAGE_SIZE (1ul << IOMMU_PAGE_SHIFT) +/* A PTE is 4 bytes */ +#define PTE_SIZE_SHIFT 2 +/* L1/L2 table sizing (IOMMU hardware pages): 1024 entries per page */ +#define PTES_PER_IOPG_SHIFT (IOMMU_PAGE_SHIFT - PTE_SIZE_SHIFT) + +#define MMMU_CTRL_OFFSET 0x00 +#define MMMU_CTRL_CAP_EXCEEDED BIT(27) +#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN BIT(26) +#define MMMU_CTRL_CAP_EXCEEDED_INT_EN BIT(25) +#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN BIT(24) +#define MMMU_CTRL_PT_INVALID BIT(20) +#define MMMU_CTRL_PT_INVALID_ABORT_EN BIT(19) +#define MMMU_CTRL_PT_INVALID_INT_EN BIT(18) +#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN BIT(17) +#define MMMU_CTRL_PT_INVALID_EN BIT(16) +#define MMMU_CTRL_WRITE_VIOLATION BIT(12) +#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN BIT(11) +#define MMMU_CTRL_WRITE_VIOLATION_INT_EN BIT(10) +#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9) +#define MMMU_CTRL_BYPASS BIT(8) +#define MMMU_CTRL_TLB_CLEARING BIT(7) +#define MMMU_CTRL_STATS_CLEAR BIT(3) +#define MMMU_CTRL_TLB_CLEAR BIT(2) +#define MMMU_CTRL_STATS_ENABLE BIT(1) +#define MMMU_CTRL_ENABLE BIT(0) + +#define MMMU_CTRL_OPERATING_FLAGS (\ + MMMU_CTRL_CAP_EXCEEDED_ABORT_EN | \ + MMMU_CTRL_PT_INVALID_ABORT_EN | \ + MMMU_CTRL_PT_INVALID_EN | \ + MMMU_CTRL_WRITE_VIOLATION_ABORT_EN | \ + MMMU_CTRL_STATS_ENABLE | \ + MMMU_CTRL_ENABLE) + +#define MMMU_PT_PA_BASE_OFFSET 0x04 + +#define MMMU_ADDR_CAP_OFFSET 0x14 +#define MMMU_ADDR_CAP_ENABLE BIT(31) +#define ADDR_CAP_SHIFT ilog2(SZ_256M) + +#define MMMU_SHOOT_DOWN_OFFSET 0x18 +#define MMMU_SHOOT_DOWN_SHOOTING BIT(31) +#define MMMU_SHOOT_DOWN_SHOOT BIT(30) + +#define MMMU_BYPASS_START_OFFSET 0x1c +#define MMMU_BYPASS_START_ENABLE BIT(31) + +#define MMMU_BYPASS_END_OFFSET 0x20 +#define MMMU_BYPASS_END_ENABLE BIT(31) + +#define MMMU_MISC_OFFSET 0x24 +#define MMMU_MISC_SINGLE_TABLE BIT(31) + +#define MMMU_ILLEGAL_ADR_OFFSET 0x30 +#define MMMU_ILLEGAL_ADR_ENABLE BIT(31) + +#define MMMU_DEBUG_INFO_OFFSET 0x38 +#define MMMU_DEBUG_INFO_VERSION_MASK 0x0000000Fu +#define MMMU_DEBUG_INFO_VA_WIDTH_MASK 0x000000F0u +#define MMMU_DEBUG_INFO_PA_WIDTH_MASK 0x00000F00u +#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK 0x000FF000u +#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK 0x0FF00000u +#define MMMU_DEBUG_INFO_BYPASS_4M BIT(28) +#define MMMU_DEBUG_INFO_BYPASS BIT(29) + +struct bcm2712_iommu { + struct device *dev; + struct iommu_device iommu; + struct bcm2712_iommu_domain *domain; + struct bcm2712_iommu_cache *cache; + void __iomem *reg_base; + spinlock_t hw_lock; + size_t bigpage_size; + size_t superpage_size; +}; + +struct bcm2712_iommu_domain { + union { + struct iommu_domain base; + struct pt_iommu_bcm2712 pt; + }; + struct bcm2712_iommu *mmu; + void *default_page; +}; + +static inline void bcm2712_iommu_writel(struct bcm2712_iommu *mmu, + unsigned int offset, u32 val) +{ + writel(val, mmu->reg_base + offset); +} + +static inline u32 bcm2712_iommu_readl(struct bcm2712_iommu *mmu, + unsigned int offset) +{ + return readl(mmu->reg_base + offset); +} + +static struct bcm2712_iommu_domain * +to_bcm2712_domain(struct iommu_domain *domain) +{ + return container_of(domain, struct bcm2712_iommu_domain, base); +} + +static void bcm2712_iommu_init(struct bcm2712_iommu *mmu) +{ + unsigned int bigpage_width, superpage_width; + u32 u = bcm2712_iommu_readl(mmu, MMMU_DEBUG_INFO_OFFSET); + u32 pa_width = FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u); + + dev_dbg(mmu->dev, "DEBUG_INFO = 0x%08x\n", u); + WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 || + FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 || + pa_width < 6 || !(u & MMMU_DEBUG_INFO_BYPASS)); + + dma_set_mask_and_coherent(mmu->dev, DMA_BIT_MASK(pa_width + 30u)); + + bigpage_width = FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u); + if (bigpage_width) + mmu->bigpage_size = IOMMU_PAGE_SIZE << bigpage_width; + + superpage_width = FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u); + if (superpage_width) + mmu->superpage_size = IOMMU_PAGE_SIZE << superpage_width; + + /* Disable MMU and clear sticky flags */ + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, + MMMU_CTRL_CAP_EXCEEDED | MMMU_CTRL_PT_INVALID | + MMMU_CTRL_WRITE_VIOLATION | + MMMU_CTRL_STATS_CLEAR); + + /* Put MMU into 2-level mode */ + bcm2712_iommu_writel(mmu, MMMU_MISC_OFFSET, + bcm2712_iommu_readl(mmu, MMMU_MISC_OFFSET) & + ~MMMU_MISC_SINGLE_TABLE); +} + +static int bcm2712_iommu_identity_attach(struct iommu_domain *identity_domain, + struct device *dev, + struct iommu_domain *old) +{ + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev); + + guard(spinlock_irqsave)(&mmu->hw_lock); + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0); + mmu->domain = NULL; + + return 0; +} + +static struct iommu_domain bcm2712_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &(const struct iommu_domain_ops) { + .attach_dev = bcm2712_iommu_identity_attach, + }, +}; + +static int bcm2712_iommu_clear_and_enable(struct bcm2712_iommu *mmu) + __must_hold(&mmu->hw_lock) +{ + u32 val; + + bcm2712_iommu_cache_flush(mmu->cache); + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, + MMMU_CTRL_OPERATING_FLAGS | MMMU_CTRL_TLB_CLEAR); + return readl_poll_timeout_atomic(mmu->reg_base + MMMU_CTRL_OFFSET, val, + !(val & MMMU_CTRL_TLB_CLEARING), 0, + 50); +} + + +static int bcm2712_iommu_attach_dev(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev); + struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain); + struct pt_iommu_bcm2712_hw_info info; + u32 default_page_pfn; + int ret = 0; + + /* all attached devices must belong to the same IOMMU instance */ + if (mydomain->mmu != mmu) + return -EINVAL; + + scoped_guard(spinlock_irqsave, &mmu->hw_lock) { + if (mmu->domain == mydomain) + break; + + /* Configure translation aperture */ + bcm2712_iommu_writel(mmu, MMMU_ADDR_CAP_OFFSET, + MMMU_ADDR_CAP_ENABLE | + (domain->geometry.aperture_end >> + ADDR_CAP_SHIFT)); + + /* + * When the IOMMU handles a request, it adds the + * PT_PA_BASE_OFFSET to (IOVA>>32) to calculate the PFN of the + * corresponding L1 directory page. + * IOVA bits [31:22] are then used to fetch the L1 descriptor + * within (which in turn points to the L2 table). + * This clever logic would allow for a L1 table larger than 4kb + * (and hence a larger aperture). + */ + pt_iommu_bcm2712_hw_info(&mydomain->pt, &info); + bcm2712_iommu_writel(mmu, MMMU_PT_PA_BASE_OFFSET, + info.pt_base >> IOMMU_PAGE_SHIFT); + + /* Disable bypass window */ + bcm2712_iommu_writel(mmu, MMMU_BYPASS_START_OFFSET, 0); + bcm2712_iommu_writel(mmu, MMMU_BYPASS_END_OFFSET, 0); + + /* + * A default (error) page is used to catch illegal reads/writes. + */ + default_page_pfn = virt_to_phys(mydomain->default_page) >> + IOMMU_PAGE_SHIFT; + bcm2712_iommu_writel(mmu, MMMU_ILLEGAL_ADR_OFFSET, + MMMU_ILLEGAL_ADR_ENABLE | + default_page_pfn); + + ret = bcm2712_iommu_clear_and_enable(mmu); + if (ret) { + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0); + mmu->domain = NULL; + break; + } + + mmu->domain = mydomain; + } + + if (ret) + dev_err_ratelimited(mmu->dev, + "TLB clear timed out during attach\n"); + + return ret; +} + +static int bcm2712_iommu_shootdown_range(struct bcm2712_iommu *mmu, + unsigned long iova, size_t size) + __must_hold(&mmu->hw_lock) +{ + unsigned long start_pfn = iova >> IOMMU_PAGE_SHIFT; + unsigned long end_pfn = (iova + size - 1) >> IOMMU_PAGE_SHIFT; + unsigned long pfn; + u32 val; + int ret; + + /* Shootdown register deals with 4 pages at a time */ + for (pfn = round_down(start_pfn, 4); pfn <= end_pfn; pfn += 4) { + bcm2712_iommu_writel(mmu, MMMU_SHOOT_DOWN_OFFSET, + MMMU_SHOOT_DOWN_SHOOT | pfn); + ret = readl_poll_timeout_atomic( + mmu->reg_base + MMMU_SHOOT_DOWN_OFFSET, val, + !(val & MMMU_SHOOT_DOWN_SHOOTING), 0, 50); + if (ret) + return ret; + } + + return 0; +} + +static int bcm2712_iommu_sync_range(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain); + struct bcm2712_iommu *mmu = mydomain->mmu; + int ret = 0; + + if (!size) + return 0; + + scoped_guard(spinlock_irqsave, &mmu->hw_lock) { + if (mmu->domain != mydomain) + return 0; + + /* If invalidating more than 1MB, just do a full TLB clear */ + if (size >= SZ_1M) { + ret = bcm2712_iommu_clear_and_enable(mmu); + } else { + bcm2712_iommu_cache_flush(mmu->cache); + ret = bcm2712_iommu_shootdown_range(mmu, iova, size); + } + } + + if (ret) + dev_err_ratelimited(mmu->dev, + "TLB sync timed out (size=%#zx)\n", size); + + return ret; +} + +static void bcm2712_iommu_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + bcm2712_iommu_sync_range(domain, gather->start, + gather->end - gather->start + 1); + iommu_put_pages_list(&gather->freelist); +} + +static int bcm2712_iommu_sync_map(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + return bcm2712_iommu_sync_range(domain, iova, size); +} + +static void bcm2712_iommu_sync_all(struct iommu_domain *domain) +{ + size_t aperture_size = domain->geometry.aperture_end - + domain->geometry.aperture_start + 1; + + bcm2712_iommu_sync_range(domain, domain->geometry.aperture_start, + aperture_size); +} + +static void bcm2712_iommu_domain_free(struct iommu_domain *domain) +{ + struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain); + struct bcm2712_iommu *mmu = mydomain->mmu; + + scoped_guard(spinlock_irqsave, &mmu->hw_lock) { + if (mmu->domain == mydomain) { + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0); + mmu->domain = NULL; + } + } + + pt_iommu_deinit(&mydomain->pt.iommu); + if (mydomain->default_page) + iommu_free_pages(mydomain->default_page); + kfree(mydomain); +} + +static const struct iommu_domain_ops bcm2712_paging_domain_ops = { + IOMMU_PT_DOMAIN_OPS(bcm2712), + .attach_dev = bcm2712_iommu_attach_dev, + .iotlb_sync = bcm2712_iommu_sync, + .iotlb_sync_map = bcm2712_iommu_sync_map, + .flush_iotlb_all = bcm2712_iommu_sync_all, + .free = bcm2712_iommu_domain_free, +}; + +static struct iommu_domain *bcm2712_iommu_domain_alloc(struct device *dev) +{ + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev); + struct bcm2712_iommu_domain *domain; + struct pt_iommu_bcm2712_cfg cfg = {}; + int ret; + + domain = kzalloc_obj(*domain); + if (!domain) + return NULL; + + domain->mmu = mmu; + domain->pt.iommu.iommu_device = mmu->dev; + cfg.common.features = BIT(PT_FEAT_DMA_INCOHERENT); + + /* Bigpage and superpage sizes are typically 64K and 1M, but may vary */ + if (mmu->bigpage_size) + cfg.bigpage_lg2 = ilog2(mmu->bigpage_size); + if (mmu->superpage_size) + cfg.superpage_lg2 = ilog2(mmu->superpage_size); + + /* 2-level format: 10-bit L1 + 10-bit L2 + 12-bit page offset */ + cfg.common.hw_max_vasz_lg2 = + (2 * PTES_PER_IOPG_SHIFT) + IOMMU_PAGE_SHIFT; + + /* PTEs encode a 25-bit output address PFN */ + cfg.common.hw_max_oasz_lg2 = 25 + IOMMU_PAGE_SHIFT; + + ret = pt_iommu_bcm2712_init(&domain->pt, &cfg, GFP_KERNEL); + if (ret) + goto err; + + /* Set up a default (error) page used to catch illegal reads/writes */ + domain->default_page = iommu_alloc_pages_sz(GFP_KERNEL, PAGE_SIZE); + if (!domain->default_page) + goto err; + + domain->base.ops = &bcm2712_paging_domain_ops; + return &domain->base; + +err: + bcm2712_iommu_domain_free(&domain->base); + return NULL; +} + +static struct bcm2712_iommu * +bcm2712_iommu_get_by_fwnode(struct fwnode_handle *fwnode) +{ + struct device *dev __free(put_device) = + bus_find_device_by_fwnode(&platform_bus_type, fwnode); + + return dev ? dev_get_drvdata(dev) : NULL; +} + +static struct iommu_device *bcm2712_iommu_probe_device(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct bcm2712_iommu *mmu; + + if (!fwspec || !fwspec->iommu_fwnode) + return ERR_PTR(-ENODEV); + + mmu = bcm2712_iommu_get_by_fwnode(fwspec->iommu_fwnode); + if (!mmu) + return ERR_PTR(-ENODEV); + + dev_iommu_priv_set(dev, mmu); + + return &mmu->iommu; +} + +static int bcm2712_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) +{ + return iommu_fwspec_add_ids(dev, args->args, 0); +} + +static const struct iommu_ops bcm2712_iommu_ops = { + .identity_domain = &bcm2712_identity_domain, + .domain_alloc_paging = bcm2712_iommu_domain_alloc, + .probe_device = bcm2712_iommu_probe_device, + .device_group = generic_single_device_group, + .get_resv_regions = iommu_dma_get_resv_regions, + .of_xlate = bcm2712_iommu_of_xlate, +}; + +static const struct of_device_id bcm2712_iommu_of_match[] = { + { .compatible = "brcm,bcm2712-iommu" }, + { /* sentinel */ } +}; + +static int bcm2712_iommu_init_cache(struct bcm2712_iommu *mmu, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + struct device_node *cache_np __free(device_node) = + of_parse_phandle(dev->of_node, "brcm,iommu-cache", 0); + if (!cache_np) + return dev_err_probe(dev, -ENOENT, + "missing brcm,iommu-cache property\n"); + + struct platform_device *cache_pdev __free(platform_device_put) = + of_find_device_by_node(cache_np); + if (!cache_pdev) + return dev_err_probe(dev, -EPROBE_DEFER, + "waiting for cache device\n"); + + mmu->cache = platform_get_drvdata(cache_pdev); + if (!mmu->cache) + return dev_err_probe(dev, -EPROBE_DEFER, + "waiting for cache driver probe\n"); + + return 0; +} + +static int bcm2712_iommu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct bcm2712_iommu *mmu; + int ret; + + mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL); + if (!mmu) + return -ENOMEM; + + mmu->dev = dev; + spin_lock_init(&mmu->hw_lock); + + mmu->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mmu->reg_base)) + return PTR_ERR(mmu->reg_base); + + ret = bcm2712_iommu_init_cache(mmu, pdev); + if (ret) + return ret; + + platform_set_drvdata(pdev, mmu); + bcm2712_iommu_init(mmu); + + ret = iommu_device_sysfs_add(&mmu->iommu, dev, NULL, "%s", + dev_name(dev)); + if (ret) + return ret; + + ret = iommu_device_register(&mmu->iommu, &bcm2712_iommu_ops, dev); + if (ret) { + iommu_device_sysfs_remove(&mmu->iommu); + return ret; + } + + return 0; +} + +static struct platform_driver bcm2712_iommu_driver = { + .driver = { + .name = "bcm2712-iommu", + .of_match_table = bcm2712_iommu_of_match, + .suppress_bind_attrs = true, + }, + .probe = bcm2712_iommu_probe, +}; +builtin_platform_driver(bcm2712_iommu_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Daniel Drake <dan@reactivated.net>"); +MODULE_DESCRIPTION("Broadcom BCM2712 IOMMU driver");
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 68ee578..78742ae 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c
@@ -37,6 +37,7 @@ #include "dma-iommu.h" #include "iommu-pages.h" +#include "iommu-priv.h" struct iommu_dma_msi_page { struct list_head list; @@ -1477,6 +1478,11 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, sg_dma_len(s) = s_length; s->offset -= s_iova_off; s_length = iova_align(iovad, s_length + s_iova_off); + if (overflows_type(s_length, s->length)) { + ret = -EOVERFLOW; + + goto out_restore_sg; + } s->length = s_length; /* @@ -1493,7 +1499,20 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, * time through here (i.e. before it has a meaningful value). */ if (pad_len && pad_len < s_length - 1) { - prev->length += pad_len; + unsigned int new_prev_len; + + /* + * For large mappings spanning multiple GBs we + * may not be able to fit all needed padding into + * sg->length. + */ + if (check_add_overflow(prev->length, pad_len, &new_prev_len)) { + ret = -EOVERFLOW; + + goto out_restore_sg; + } + + prev->length = new_prev_len; iova_len += pad_len; } @@ -1628,10 +1647,12 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size, void *cpu_addr; page = dma_alloc_contiguous(dev, alloc_size, gfp); - if (!page) - page = alloc_pages_node(node, gfp, get_order(alloc_size)); - if (!page) - return NULL; + if (!page) { + cpu_addr = alloc_pages_exact_nid(node, alloc_size, gfp); + if (!cpu_addr) + return NULL; + page = virt_to_page(cpu_addr); + } if (!coherent || PageHighMem(page)) { pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); @@ -1762,7 +1783,7 @@ unsigned long iommu_dma_get_merge_boundary(struct device *dev) return (1UL << __ffs(domain->pgsize_bitmap)) - 1; } -size_t iommu_dma_opt_mapping_size(void) +size_t iommu_dma_max_opt_mapping_size(void) { return iova_rcache_range(); } @@ -2161,12 +2182,6 @@ void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain) dev_clear_dma_iommu(dev); } -static bool has_msi_cookie(const struct iommu_domain *domain) -{ - return domain && (domain->cookie_type == IOMMU_COOKIE_DMA_IOVA || - domain->cookie_type == IOMMU_COOKIE_DMA_MSI); -} - static size_t cookie_msi_granule(const struct iommu_domain *domain) { switch (domain->cookie_type) { @@ -2248,11 +2263,6 @@ int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc, struct device *dev = msi_desc_to_dev(desc); const struct iommu_dma_msi_page *msi_page; - if (!has_msi_cookie(domain)) { - msi_desc_set_iommu_msi_iova(desc, 0, 0); - return 0; - } - iommu_group_mutex_assert(dev); msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); if (!msi_page)
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 874d05f..0bca366 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c
@@ -8,6 +8,7 @@ #define DEBUG #endif +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/dma-mapping.h> #include <linux/err.h> @@ -143,6 +144,10 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CTRL_VM_FAULT_MODE_STALL BIT(3) #define CAPA0_CAPA1_EXIST BIT(11) #define CAPA1_VCR_ENABLED BIT(14) +#define CAPA1_NO_BLOCK_MODE BIT(15) +#define FAULT_INFO_AXID GENMASK(15, 0) +#define FAULT_INFO_AXLEN GENMASK(19, 16) +#define FAULT_INFO_WRITE BIT(20) /* common registers */ #define REG_MMU_CTRL 0x000 @@ -193,6 +198,7 @@ struct sysmmu_fault { sysmmu_iova_t addr; /* IOVA address that caused fault */ const char *name; /* human readable fault name */ unsigned int type; /* fault type for report_iommu_fault() */ + u32 info; /* raw transaction info (v7+ only) */ }; struct sysmmu_v1_fault_info { @@ -306,6 +312,7 @@ struct sysmmu_drvdata { /* v7 fields */ bool has_vcr; /* virtual machine control register */ + bool no_block; /* BLOCK mode not implemented */ }; #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg) @@ -358,7 +365,8 @@ static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data, fault->addr = readl(SYSMMU_REG(data, fault_va)); fault->name = sysmmu_v7_fault_names[itype % 4]; - fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + fault->type = (info & FAULT_INFO_WRITE) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + fault->info = info; return 0; } @@ -511,6 +519,7 @@ static void __sysmmu_get_vcr(struct sysmmu_drvdata *data) u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1); data->has_vcr = capa1 & CAPA1_VCR_ENABLED; + data->no_block = capa1 & CAPA1_NO_BLOCK_MODE; } static void __sysmmu_get_version(struct sysmmu_drvdata *data) @@ -555,6 +564,12 @@ static void show_fault_information(struct sysmmu_drvdata *data, dev_name(data->master), fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE", fault->name, fault->addr); + /* AxID identifies the issuing port inside the master */ + if (data->variant->fault_info) + dev_err(data->sysmmu, "transaction info %#010x: AxID %#lx, AxLEN %lu\n", + fault->info, + FIELD_GET(FAULT_INFO_AXID, fault->info), + FIELD_GET(FAULT_INFO_AXLEN, fault->info)); dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); ent = section_entry(phys_to_virt(data->pgtable), fault->addr); dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); @@ -652,7 +667,12 @@ static void __sysmmu_enable(struct sysmmu_drvdata *data) __sysmmu_enable_clocks(data); spin_lock_irqsave(&data->lock, flags); - writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + /* + * On no-block hardware CTRL_BLOCK acts as a plain enable; keep the + * MMU disabled until it is fully programmed. + */ + if (!data->no_block) + writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); __sysmmu_init_config(data); __sysmmu_set_ptbase(data, data->pgtable); __sysmmu_enable_vid(data); @@ -677,7 +697,14 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, spin_lock_irqsave(&data->lock, flags); if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { clk_enable(data->clk_master); - if (sysmmu_block(data)) { + /* + * No-block hardware accepts invalidation writes while the + * MMU is running; it is also v7+, so flush-all covers the + * FLPD cache. + */ + if (data->no_block) { + __sysmmu_tlb_invalidate(data); + } else if (sysmmu_block(data)) { if (data->version >= MAKE_MMU_VER(5, 0)) __sysmmu_tlb_invalidate(data); else @@ -713,7 +740,9 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, if (MMU_MAJ_VER(data->version) == 2) num_inv = min_t(unsigned int, size / SPAGE_SIZE, 64); - if (sysmmu_block(data)) { + if (data->no_block) { + __sysmmu_tlb_invalidate_entry(data, iova, num_inv); + } else if (sysmmu_block(data)) { __sysmmu_tlb_invalidate_entry(data, iova, num_inv); sysmmu_unblock(data); }
diff --git a/drivers/iommu/generic_pt/.kunitconfig b/drivers/iommu/generic_pt/.kunitconfig index 0bb98fe..a3da743 100644 --- a/drivers/iommu/generic_pt/.kunitconfig +++ b/drivers/iommu/generic_pt/.kunitconfig
@@ -7,6 +7,7 @@ CONFIG_IOMMU_PT_VTDSS=y CONFIG_IOMMU_PT_RISCV64=y CONFIG_IOMMU_PT_X86_64=y +CONFIG_IOMMU_PT_BCM2712=y CONFIG_IOMMU_PT_KUNIT_TEST=y CONFIG_IOMMUFD=y
diff --git a/drivers/iommu/generic_pt/Kconfig b/drivers/iommu/generic_pt/Kconfig index f4ed1ad..2a81dac 100644 --- a/drivers/iommu/generic_pt/Kconfig +++ b/drivers/iommu/generic_pt/Kconfig
@@ -72,6 +72,15 @@ Selected automatically by an IOMMU driver that uses this format. +config IOMMU_PT_BCM2712 + tristate "IOMMU page table for Broadcom BCM2712" + help + iommu_domain implementation for the Broadcom BCM2712 IOMMU found on + Raspberry Pi 5. It supports 4K page sizes over a 2-level page table + format. + + Selected automatically by an IOMMU driver that uses this format. + config IOMMU_PT_KUNIT_TEST tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS depends on KUNIT @@ -79,6 +88,7 @@ depends on IOMMU_PT_RISCV64 || !IOMMU_PT_RISCV64 depends on IOMMU_PT_X86_64 || !IOMMU_PT_X86_64 depends on IOMMU_PT_VTDSS || !IOMMU_PT_VTDSS + depends on IOMMU_PT_BCM2712 || !IOMMU_PT_BCM2712 default KUNIT_ALL_TESTS help Enable kunit tests for GENERIC_PT and IOMMU_PT that covers all the
diff --git a/drivers/iommu/generic_pt/fmt/Makefile b/drivers/iommu/generic_pt/fmt/Makefile index e308760..a43f3b8 100644 --- a/drivers/iommu/generic_pt/fmt/Makefile +++ b/drivers/iommu/generic_pt/fmt/Makefile
@@ -9,6 +9,8 @@ iommu_pt_fmt-$(CONFIG_IOMMU_PT_X86_64) += x86_64 +iommu_pt_fmt-$(CONFIG_IOMMU_PT_BCM2712) += bcm2712 + IOMMU_PT_KUNIT_TEST := define create_format obj-$(2) += iommu_$(1).o
diff --git a/drivers/iommu/generic_pt/fmt/bcm2712.h b/drivers/iommu/generic_pt/fmt/bcm2712.h new file mode 100644 index 0000000..2c5b034 --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/bcm2712.h
@@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Daniel Drake + * + * BCM2712 IOMMU simple two level page table + */ +#ifndef __GENERIC_PT_FMT_BCM2712_H +#define __GENERIC_PT_FMT_BCM2712_H + +#include "defs_bcm2712.h" +#include "../pt_defs.h" + +#include <linux/bitfield.h> +#include <linux/container_of.h> +#include <linux/log2.h> +#include <linux/sizes.h> + +enum { + /* Hardware provides a two-level page table */ + PT_MAX_TOP_LEVEL = 1, + + /* Hardware page size is strictly 4kb */ + PT_GRANULE_LG2SZ = ilog2(SZ_4K), + PT_TABLEMEM_LG2SZ = PT_GRANULE_LG2SZ, + + /* Table entries and leaf entries are 32 bits */ + PT_ITEM_WORD_SIZE = sizeof(pt_bcm2712_entry_t), + + /* Leaf entries encode a 25-bit PFN */ + PT_MAX_OUTPUT_ADDRESS_LG2 = 25 + PT_GRANULE_LG2SZ, + + /* + * For simplicity, only manage mappings within an address space of + * exactly 4GB. This is because our Level 1 directory page will be 4kb + * in the smallest case, permitting 1024 entries pointing at Level 0 + * pages, each permitting 4mb of mapped memory. + */ + PT_MAX_VA_ADDRESS_LG2 = ilog2(SZ_4G), + + /* Level 1 base address is programmed as a 32-bit PFN */ + PT_TOP_PHYS_MASK = GENMASK_ULL(31 + PT_GRANULE_LG2SZ, PT_GRANULE_LG2SZ), +}; + +/* PTE bits */ +enum { + BCM2712PT_VALID = BIT(28), + BCM2712PT_WRITE = BIT(29), + BCM2712PT_PAGESIZE = GENMASK(31, 30), + BCM2712PT_PFN = GENMASK(24, 0), +}; + +#define common_to_bcm2712pt(common_ptr) \ + container_of_const(common_ptr, struct pt_bcm2712, common) +#define to_bcm2712pt(pts) common_to_bcm2712pt((pts)->range->common) + +static inline pt_oaddr_t bcm2712pt_table_pa(const struct pt_state *pts) +{ + return oalog2_mul(FIELD_GET(BCM2712PT_PFN, pts->entry), + PT_GRANULE_LG2SZ); +} +#define pt_table_pa bcm2712pt_table_pa +#define pt_item_oa bcm2712pt_table_pa + +static inline bool bcm2712pt_can_have_leaf(const struct pt_state *pts) +{ + return true; +} +#define pt_can_have_leaf bcm2712pt_can_have_leaf + +/* 4MB pages are installed at level 1, everything else at level 0 */ +static inline unsigned int bcm2712pt_pgsz_lg2_to_level(struct pt_common *common, + unsigned int pgsize_lg2) +{ + return pgsize_lg2 == ilog2(SZ_4M) ? 1 : 0; +} +#define pt_pgsz_lg2_to_level bcm2712pt_pgsz_lg2_to_level + +static inline pt_vaddr_t bcm2712pt_possible_sizes(const struct pt_state *pts) +{ + struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common); + + if (pts->level == 1) + return SZ_4M; + + return SZ_4K | + (table->bigpage_lg2 ? BIT_ULL(table->bigpage_lg2) : 0) | + (table->superpage_lg2 ? BIT_ULL(table->superpage_lg2) : 0); +} +#define pt_possible_sizes bcm2712pt_possible_sizes + +static inline unsigned int +bcm2712pt_entry_num_contig_lg2(const struct pt_state *pts) +{ + struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common); + u32 pgsz = FIELD_GET(BCM2712PT_PAGESIZE, pts->entry); + + if (pts->level != 0 || !(pts->entry & BCM2712PT_VALID)) + return 0; + + /* + * Superpage/bigpage contiguous mapping (really just hinting) is handled + * via PAGESIZE bits on each leaf entry. + */ + if (pgsz == 2 && table->superpage_lg2) + return table->superpage_lg2 - PT_GRANULE_LG2SZ; + else if (pgsz == 1 && table->bigpage_lg2) + return table->bigpage_lg2 - PT_GRANULE_LG2SZ; + return 0; +} +#define pt_entry_num_contig_lg2 bcm2712pt_entry_num_contig_lg2 + +static inline unsigned int bcm2712pt_num_items_lg2(const struct pt_state *pts) +{ + return PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE); +} +#define pt_num_items_lg2 bcm2712pt_num_items_lg2 + +static inline enum pt_entry_type bcm2712pt_load_entry_raw(struct pt_state *pts) +{ + const pt_bcm2712_entry_t *tablep = + pt_cur_table(pts, pt_bcm2712_entry_t); + + pts->entry = READ_ONCE(tablep[pts->index]); + if (!(pts->entry & BCM2712PT_VALID)) + return PT_ENTRY_EMPTY; + + if (pts->level == 1) { + if (FIELD_GET(BCM2712PT_PAGESIZE, pts->entry) == 3) + return PT_ENTRY_OA; + return PT_ENTRY_TABLE; + } + + return PT_ENTRY_OA; +} +#define pt_load_entry_raw bcm2712pt_load_entry_raw + +static inline void +bcm2712pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa, + unsigned int oasz_lg2, + const struct pt_write_attrs *attrs) +{ + pt_bcm2712_entry_t *tablep = pt_cur_table(pts, pt_bcm2712_entry_t); + pt_bcm2712_entry_t entry; + + if (!pt_check_install_leaf_args(pts, oa, oasz_lg2)) + return; + + entry = BCM2712PT_VALID | attrs->descriptor_bits | + FIELD_PREP(BCM2712PT_PFN, oalog2_div(oa, PT_GRANULE_LG2SZ)); + + if (pts->level == 1) { + /* Level 1 hugepage (4MB) */ + entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 3); + WRITE_ONCE(tablep[pts->index], entry); + pts->entry = entry; + return; + } + + if (oasz_lg2 == PT_GRANULE_LG2SZ) { + WRITE_ONCE(tablep[pts->index], entry); + pts->entry = entry; + } else { + struct pt_bcm2712 *table = + common_to_bcm2712pt(pts->range->common); + u32 *end; + + tablep += pts->index; + end = tablep + log2_to_int(oasz_lg2 - PT_GRANULE_LG2SZ); + + /* + * Leaf entries can contain hints indicating bigpage/superpage + * contiguous mappings to permit TLB optimization + */ + if (oasz_lg2 == table->superpage_lg2) + entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 2); + else if (oasz_lg2 == table->bigpage_lg2) + entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 1); + + pts->entry = entry; + for (; tablep != end; tablep++, entry++) + WRITE_ONCE(*tablep, entry); + } +} +#define pt_install_leaf_entry bcm2712pt_install_leaf_entry + +static inline bool bcm2712pt_install_table(struct pt_state *pts, + pt_oaddr_t table_pa, + const struct pt_write_attrs *attrs) +{ + pt_bcm2712_entry_t entry = + BCM2712PT_VALID | + FIELD_PREP(BCM2712PT_PFN, + oalog2_div(table_pa, PT_GRANULE_LG2SZ)); + + return pt_table_install32(pts, entry); +} +#define pt_install_table bcm2712pt_install_table + +static inline void bcm2712pt_attr_from_entry(const struct pt_state *pts, + struct pt_write_attrs *attrs) +{ + attrs->descriptor_bits = pts->entry & BCM2712PT_WRITE; +} +#define pt_attr_from_entry bcm2712pt_attr_from_entry + +static inline unsigned int bcm2712pt_max_sw_bit(struct pt_common *common) +{ + return 2; +} +#define pt_max_sw_bit bcm2712pt_max_sw_bit + +static inline u32 bcm2712pt_sw_bit(unsigned int bitnr) +{ + if (__builtin_constant_p(bitnr) && bitnr > 2) + BUILD_BUG(); + if (PT_WARN_ON(bitnr > 2)) + return 0; + + /* Bits 27, 26 and 25 are believed spare */ + return BIT(27 - bitnr); +} +#define pt_sw_bit bcm2712pt_sw_bit + +/* --- iommu */ +#include <linux/generic_pt/iommu.h> +#include <linux/iommu.h> + +#define pt_iommu_table pt_iommu_bcm2712 + +/* The common struct is in the per-format common struct */ +static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table) +{ + return &container_of(iommu_table, struct pt_iommu_table, iommu) + ->bcm2712pt.common; +} + +static inline struct pt_iommu *iommu_from_common(struct pt_common *common) +{ + return &container_of(common, struct pt_iommu_table, bcm2712pt.common) + ->iommu; +} + +static inline int bcm2712_pt_iommu_set_prot(struct pt_common *common, + struct pt_write_attrs *attrs, + unsigned int iommu_prot) +{ + attrs->descriptor_bits = 0; + if (iommu_prot & IOMMU_WRITE) + attrs->descriptor_bits |= BCM2712PT_WRITE; + return 0; +} +#define pt_iommu_set_prot bcm2712_pt_iommu_set_prot + +static inline int bcm2712_pt_fmt_init(struct pt_iommu_table *fmt_table, + const struct pt_iommu_bcm2712_cfg *cfg) +{ + fmt_table->bcm2712pt.bigpage_lg2 = cfg->bigpage_lg2; + fmt_table->bcm2712pt.superpage_lg2 = cfg->superpage_lg2; + + pt_top_set_level(&fmt_table->bcm2712pt.common, PT_MAX_TOP_LEVEL); + return 0; +} +#define pt_iommu_fmt_init bcm2712_pt_fmt_init + +static inline void +bcm2712pt_iommu_fmt_hw_info(struct pt_iommu_bcm2712 *table, + const struct pt_range *top_range, + struct pt_iommu_bcm2712_hw_info *info) +{ + info->pt_base = virt_to_phys(top_range->top_table); + PT_WARN_ON(info->pt_base & ~PT_TOP_PHYS_MASK); +} +#define pt_iommu_fmt_hw_info bcm2712pt_iommu_fmt_hw_info + +#if defined(GENERIC_PT_KUNIT) +static const struct pt_iommu_bcm2712_cfg bcm2712_kunit_fmt_cfgs[] = { + [0] = { + .common.hw_max_vasz_lg2 = 32, + .common.hw_max_oasz_lg2 = 37, + .bigpage_lg2 = 16, + .superpage_lg2 = 20, + }, +}; +#define kunit_fmt_cfgs bcm2712_kunit_fmt_cfgs +enum { KUNIT_FMT_FEATURES = 0 }; +#endif + +#endif
diff --git a/drivers/iommu/generic_pt/fmt/defs_bcm2712.h b/drivers/iommu/generic_pt/fmt/defs_bcm2712.h new file mode 100644 index 0000000..f1d983b --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/defs_bcm2712.h
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __GENERIC_PT_FMT_DEFS_BCM2712_H +#define __GENERIC_PT_FMT_DEFS_BCM2712_H + +#include <linux/generic_pt/common.h> +#include <linux/types.h> + +typedef u32 pt_bcm2712_entry_t; +typedef u64 pt_vaddr_t; +typedef u64 pt_oaddr_t; + +struct bcm2712pt_write_attrs { + pt_bcm2712_entry_t descriptor_bits; + gfp_t gfp; +}; +#define pt_write_attrs bcm2712pt_write_attrs + +#endif
diff --git a/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c b/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c new file mode 100644 index 0000000..43fbc3c --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c
@@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define PT_FMT bcm2712 +#define PT_SUPPORTED_FEATURES BIT(PT_FEAT_DMA_INCOHERENT) + +#include <linux/generic_pt/iommu.h> +#include "iommu_template.h"
diff --git a/drivers/iommu/generic_pt/kunit_iommu_pt.h b/drivers/iommu/generic_pt/kunit_iommu_pt.h index ece1c9b..9fc71db 100644 --- a/drivers/iommu/generic_pt/kunit_iommu_pt.h +++ b/drivers/iommu/generic_pt/kunit_iommu_pt.h
@@ -403,7 +403,7 @@ static void test_pgsize_boundary(struct kunit *test) priv->smallest_pgsz != SZ_4K) kunit_skip(test, "Format does not have the required range"); - do_map(test, 0xfef80000, 0x208b95d000, 0xfef9ffff - 0xfef80000 + 1); + do_map(test, 0xfef80000, 0x8b95d000, 0xfef9ffff - 0xfef80000 + 1); } /* See https://lore.kernel.org/r/20250826143816.38686-1-eugkoira@amazon.com */
diff --git a/drivers/iommu/hyperv/Kconfig b/drivers/iommu/hyperv/Kconfig new file mode 100644 index 0000000..f2f993f --- /dev/null +++ b/drivers/iommu/hyperv/Kconfig
@@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Hyper-V para-virtualized IOMMU driver for Linux guests +config HYPERV_PVIOMMU + bool "Microsoft Hyper-V para-virtualized IOMMU driver for Linux guests" + depends on X86_64 && HYPERV && PCI + select IOMMU_DMA + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_X86_64 + default HYPERV + help + Para-virtualized IOMMU driver for Linux guests running on + Microsoft Hyper-V. Provides DMA remapping and IOTLB + flush support to enable DMA isolation for PCI devices + assigned to the guest.
diff --git a/drivers/iommu/hyperv/Makefile b/drivers/iommu/hyperv/Makefile index 6ef0ef9..d3b2a25 100644 --- a/drivers/iommu/hyperv/Makefile +++ b/drivers/iommu/hyperv/Makefile
@@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_IRQ_REMAP) += hv-irq-remap-x86.o +obj-$(CONFIG_HYPERV_PVIOMMU) += hv-iommu-guest.o
diff --git a/drivers/iommu/hyperv/hv-iommu-guest.c b/drivers/iommu/hyperv/hv-iommu-guest.c new file mode 100644 index 0000000..876a22f --- /dev/null +++ b/drivers/iommu/hyperv/hv-iommu-guest.c
@@ -0,0 +1,682 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Hyper-V para-virtualized IOMMU driver for Linux guests. + * + * Copyright (C) 2024-2026 Microsoft, Inc. + */ + +#define pr_fmt(fmt) "Hyper-V pvIOMMU: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include <linux/iommu.h> +#include <linux/pci.h> +#include <linux/dma-map-ops.h> +#include <linux/generic_pt/iommu.h> +#include <linux/pci-ats.h> + +#include <asm/iommu.h> +#include <asm/hypervisor.h> +#include <asm/mshyperv.h> + +#include "iommu.h" +#include "../iommu-pages.h" + +static struct hv_iommu_dev *hv_iommu_device; + +/* + * Identity and blocking domains are static singletons: identity is a 1:1 + * passthrough with no page table, blocking rejects all DMA. Neither holds + * per-IOMMU state, so one instance suffices even with multiple vIOMMUs. + */ +static const struct iommu_domain_ops hv_iommu_identity_domain_ops; +static const struct iommu_domain_ops hv_iommu_blocking_domain_ops; +static struct iommu_ops hv_iommu_ops; + +static struct hv_iommu_domain hv_identity_domain = { + .domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &hv_iommu_identity_domain_ops, + .owner = &hv_iommu_ops, + }, +}; + +static struct hv_iommu_domain hv_blocking_domain = { + .domain = { + .type = IOMMU_DOMAIN_BLOCKED, + .ops = &hv_iommu_blocking_domain_ops, + .owner = &hv_iommu_ops, + }, +}; + +static inline bool hv_iommu_present(u64 cap) +{ + return cap & HV_IOMMU_CAP_PRESENT; +} + +static inline bool hv_iommu_s1_domain_supported(u64 cap) +{ + return cap & HV_IOMMU_CAP_S1; +} + +static inline bool hv_iommu_5lvl_supported(u64 cap) +{ + return cap & HV_IOMMU_CAP_S1_5LVL; +} + +static inline bool hv_iommu_ats_supported(u64 cap) +{ + return cap & HV_IOMMU_CAP_ATS; +} + +static int hv_create_device_domain(struct hv_iommu_domain *hv_domain) +{ + int ret; + u64 status; + unsigned long flags; + struct hv_input_create_device_domain *input; + + ret = ida_alloc_range(&hv_iommu_device->domain_ids, + hv_iommu_device->first_domain, + hv_iommu_device->last_domain, GFP_KERNEL); + if (ret < 0) + return ret; + + hv_domain->device_domain.partition_id = HV_PARTITION_ID_SELF; + hv_domain->device_domain.domain_id.type = HV_DEVICE_DOMAIN_TYPE_S1; + hv_domain->device_domain.domain_id.id = ret; + hv_domain->hv_iommu = hv_iommu_device; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + input->create_device_domain_flags.forward_progress_required = 1; + input->create_device_domain_flags.inherit_owning_vtl = 0; + status = hv_do_hypercall(HVCALL_CREATE_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status)) { + hv_status_err(status, "HVCALL_CREATE_DEVICE_DOMAIN failed\n"); + ida_free(&hv_iommu_device->domain_ids, + hv_domain->device_domain.domain_id.id); + } + + return hv_result_to_errno(status); +} + +static void hv_delete_device_domain(struct hv_iommu_domain *hv_domain) +{ + u64 status; + unsigned long flags; + struct hv_input_delete_device_domain *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + status = hv_do_hypercall(HVCALL_DELETE_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status)) + hv_status_err(status, "HVCALL_DELETE_DEVICE_DOMAIN failed\n"); + + ida_free(&hv_domain->hv_iommu->domain_ids, + hv_domain->device_domain.domain_id.id); +} + +static int +hv_configure_device_domain(struct hv_iommu_domain *hv_domain, + const struct hv_device_domain_settings *settings) +{ + u64 status; + unsigned long flags; + struct hv_input_configure_device_domain *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + input->settings = *settings; + status = hv_do_hypercall(HVCALL_CONFIGURE_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status)) + hv_status_err(status, + "HVCALL_CONFIGURE_DEVICE_DOMAIN failed\n"); + + return hv_result_to_errno(status); +} + +static int +hv_create_configure_device_domain(struct hv_iommu_domain *hv_domain, + const struct hv_device_domain_settings *settings) +{ + int ret; + + ret = hv_create_device_domain(hv_domain); + if (ret) + return ret; + + ret = hv_configure_device_domain(hv_domain, settings); + if (ret) + hv_delete_device_domain(hv_domain); + + return ret; +} + +static bool hv_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + return cap == IOMMU_CAP_CACHE_COHERENCY; +} + +static void hv_flush_device_domain(struct hv_iommu_domain *hv_domain) +{ + u64 status; + unsigned long flags; + struct hv_input_flush_device_domain *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + status = hv_do_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + WARN_ONCE(!hv_result_success(status), + "HVCALL_FLUSH_DEVICE_DOMAIN failed: %#llx (%s)\n", + status, hv_result_to_string(status)); +} + +static int hv_iommu_attach_dev(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + u64 status; + u32 prefix; + unsigned long flags; + struct pci_dev *pdev; + struct hv_input_attach_device_domain *input; + struct hv_iommu_domain *hv_domain = to_hv_iommu_domain(domain); + int pci_domain; + int ret; + + pdev = to_pci_dev(dev); + dev_dbg(dev, "attaching to domain %d\n", + hv_domain->device_domain.domain_id.id); + + pci_domain = pci_domain_nr(pdev->bus); + ret = hv_pci_lookup_dev_id(pci_domain, &prefix); + if (ret) { + dev_err(dev, + "no logical device ID registered for PCI domain %04x\n", + pci_domain); + return ret; + } + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + input->device_id.as_uint64 = (u64)prefix | PCI_FUNC(pdev->devfn); + status = hv_do_hypercall(HVCALL_ATTACH_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status) && + hv_result(status) != HV_STATUS_DEVICE_ALREADY_IN_DOMAIN) { + hv_status_err(status, "HVCALL_ATTACH_DEVICE_DOMAIN failed\n"); + return hv_result_to_errno(status); + } + + if (domain != &hv_blocking_domain.domain && + !pdev->ats_enabled && + hv_iommu_ats_supported(hv_domain->hv_iommu->cap) && + pci_ats_supported(pdev)) + pci_enable_ats(pdev, PAGE_SHIFT); + + return 0; +} + +static int hv_iommu_blocking_attach_dev(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + if (pdev->ats_enabled) + pci_disable_ats(pdev); + + ret = hv_iommu_attach_dev(domain, dev, old); + + /* + * Attaching to the blocking domain only asks the hypervisor to + * disable translation and IOPF for the device, so it cannot fail + * unless there is a driver or hypervisor bug. Return the hypercall + * status rather than 0 so that a failure on the DMA ownership claim + * path (VFIO/iommufd) fails the claim instead of leaving the device + * unblocked. WARN since such a failure indicates a bug. + */ + WARN_ON(ret); + return ret; +} + +static int +hv_iommu_get_dev_property(struct device *dev, u32 code, + struct hv_output_get_logical_device_property *prop) +{ + u64 status; + u32 prefix; + unsigned long flags; + int ret; + struct pci_dev *pdev = to_pci_dev(dev); + struct hv_input_get_logical_device_property *input; + struct hv_output_get_logical_device_property *output; + + ret = hv_pci_lookup_dev_id(pci_domain_nr(pdev->bus), &prefix); + if (ret) + return ret; + + local_irq_save(flags); + + /* + * The combined input and output fit in the per-CPU input page, + * so no separate output page is needed. + */ + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = (struct hv_output_get_logical_device_property *)(input + 1); + memset(input, 0, sizeof(*input)); + input->partition_id = HV_PARTITION_ID_SELF; + input->logical_device_id = (u64)prefix | PCI_FUNC(pdev->devfn); + input->code = code; + status = hv_do_hypercall(HVCALL_GET_LOGICAL_DEVICE_PROPERTY, + input, output); + *prop = *output; + + local_irq_restore(flags); + + if (!hv_result_success(status)) + hv_status_err(status, + "HVCALL_GET_LOGICAL_DEVICE_PROPERTY failed\n"); + + return hv_result_to_errno(status); +} + +static struct iommu_device *hv_iommu_probe_device(struct device *dev) +{ + struct hv_output_get_logical_device_property property = {0}; + int ret; + + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + ret = hv_iommu_get_dev_property(dev, HV_LOGICAL_DEVICE_PROPERTY_PVIOMMU, + &property); + if (ret || !(property.device_iommu & HV_DEVICE_IOMMU_ENABLED)) + return ERR_PTR(-ENODEV); + + return &hv_iommu_device->iommu; +} + +static struct iommu_group *hv_iommu_device_group(struct device *dev) +{ + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + return pci_device_group(dev); +} + +static int __init hv_initialize_static_domains(void) +{ + /* + * Clearing translation_enabled bypasses stage-1 translation, so DMA + * addresses are used directly as GPAs. Hyper-V requires paging and + * blocked domains to keep translation_enabled set. + */ + const struct hv_device_domain_settings identity_settings = { + .flags.translation_enabled = 0, + }; + const struct hv_device_domain_settings blocked_settings = { + .flags = { + .translation_enabled = 1, + .blocked = 1, + }, + }; + int ret; + + /* Default stage-1 identity domain */ + ret = hv_create_configure_device_domain(&hv_identity_domain, + &identity_settings); + if (ret) + return ret; + + /* Default stage-1 blocked domain */ + ret = hv_create_configure_device_domain(&hv_blocking_domain, + &blocked_settings); + if (ret) + goto delete_identity_domain; + + return 0; + +delete_identity_domain: + hv_delete_device_domain(&hv_identity_domain); + return ret; +} + +static void hv_iommu_get_resv_regions(struct device *dev, + struct list_head *head) +{ + iommu_add_resv_x86_msi_region(head); +} + +static void hv_iommu_flush_iotlb_all(struct iommu_domain *domain) +{ + hv_flush_device_domain(to_hv_iommu_domain(domain)); +} + +/* + * Calculate the minimal power-of-two aligned range that covers [start, end] + * (end is inclusive). Returns a single (page_number, page_mask_shift) + * descriptor that may over-flush when the range is not naturally aligned. + */ +static void +hv_iommu_calc_flush_range(unsigned long start, unsigned long end, + union hv_iommu_flush_va *va) +{ + unsigned long aligned_start; + unsigned int sz_lg2; + + sz_lg2 = max(fls_long(start ^ end), HV_HYP_PAGE_SHIFT); + + /* + * A valid IOVA range shall not span bit 63. Use the maximum mask + * so the host can safely perform a full flush. + */ + if (WARN_ON_ONCE(sz_lg2 == BITS_PER_LONG)) + aligned_start = 0; + else + aligned_start = + start & GENMASK(BITS_PER_LONG - 1, sz_lg2); + + va->page_number = aligned_start >> HV_HYP_PAGE_SHIFT; + va->page_mask_shift = sz_lg2 - HV_HYP_PAGE_SHIFT; +} + +static void hv_flush_device_domain_list(struct hv_iommu_domain *hv_domain, + struct iommu_iotlb_gather *iotlb_gather) +{ + u64 status; + unsigned long flags; + struct hv_input_flush_device_domain_list *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + /* Clear the fixed header and the single range entry. */ + memset(input, 0, struct_size(input, iova_list, 1)); + + input->device_domain = hv_domain->device_domain; + input->flags |= HV_FLUSH_DEVICE_DOMAIN_LIST_IOMMU_FORMAT; + hv_iommu_calc_flush_range(iotlb_gather->start, iotlb_gather->end, + &input->iova_list[0]); + + status = hv_do_rep_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN_LIST, + 1, 0, input, NULL); + + if (WARN_ONCE(!hv_result_success(status), + "HVCALL_FLUSH_DEVICE_DOMAIN_LIST failed: %#llx (%s)\n", + status, hv_result_to_string(status))) { + /* Page-selective flush failed, fall back to full flush. */ + struct hv_input_flush_device_domain *flush_all = (void *)input; + + memset(flush_all, 0, sizeof(*flush_all)); + flush_all->device_domain = hv_domain->device_domain; + status = hv_do_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN, + flush_all, NULL); + WARN(!hv_result_success(status), + "HVCALL_FLUSH_DEVICE_DOMAIN fallback also failed: %#llx (%s)\n", + status, hv_result_to_string(status)); + } + + local_irq_restore(flags); +} + +static void hv_iommu_iotlb_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *iotlb_gather) +{ + hv_flush_device_domain_list(to_hv_iommu_domain(domain), iotlb_gather); + + /* + * The hypercall also invalidates non-leaf page-walk caches, so + * detached page-table pages can now be freed. + */ + iommu_put_pages_list(&iotlb_gather->freelist); +} + +static void hv_iommu_paging_domain_free(struct iommu_domain *domain) +{ + struct hv_iommu_domain *hv_domain = to_hv_iommu_domain(domain); + + /* Free all remaining mappings */ + pt_iommu_deinit(&hv_domain->pt_iommu); + + hv_delete_device_domain(hv_domain); + + kfree(hv_domain); +} + +static const struct iommu_domain_ops hv_iommu_identity_domain_ops = { + .attach_dev = hv_iommu_attach_dev, +}; + +static const struct iommu_domain_ops hv_iommu_blocking_domain_ops = { + .attach_dev = hv_iommu_blocking_attach_dev, +}; + +static const struct iommu_domain_ops hv_iommu_paging_domain_ops = { + .attach_dev = hv_iommu_attach_dev, + IOMMU_PT_DOMAIN_OPS(x86_64), + .flush_iotlb_all = hv_iommu_flush_iotlb_all, + .iotlb_sync = hv_iommu_iotlb_sync, + .free = hv_iommu_paging_domain_free, +}; + +static struct iommu_domain *hv_iommu_domain_alloc_paging(struct device *dev) +{ + int ret; + struct hv_iommu_domain *hv_domain; + struct pt_iommu_x86_64_hw_info pt_info; + struct pt_iommu_x86_64_cfg cfg = {}; + struct hv_device_domain_settings settings = { + .flags = { + .translation_enabled = 1, + }, + }; + + hv_domain = kzalloc_obj(*hv_domain, GFP_KERNEL); + if (!hv_domain) + return ERR_PTR(-ENOMEM); + + hv_domain->pt_iommu.nid = dev_to_node(dev); + + cfg.common.hw_max_vasz_lg2 = hv_iommu_device->max_iova_width; + cfg.common.hw_max_oasz_lg2 = 52; + cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); + /* + * Hyper-V S1 domains use a 4-level root for IOVA widths up to + * 48 bits. A 5-level root is used only for wider apertures when + * HV_IOMMU_CAP_S1_5LVL is advertised. + */ + cfg.top_level = (hv_iommu_device->max_iova_width > 48) ? 4 : 3; + + ret = pt_iommu_x86_64_init(&hv_domain->pt_iommu_x86_64, &cfg, GFP_KERNEL); + if (ret) + goto err_free; + + /* Constrain to page sizes the hypervisor supports */ + hv_domain->domain.pgsize_bitmap &= hv_iommu_device->pgsize_bitmap; + + hv_domain->domain.ops = &hv_iommu_paging_domain_ops; + + pt_iommu_x86_64_hw_info(&hv_domain->pt_iommu_x86_64, &pt_info); + settings.page_table_root = pt_info.gcr3_pt; + settings.flags.first_stage_paging_mode = pt_info.levels == 5; + + ret = hv_create_configure_device_domain(hv_domain, &settings); + if (ret) + goto err_pt_deinit; + + return &hv_domain->domain; + +err_pt_deinit: + pt_iommu_deinit(&hv_domain->pt_iommu); +err_free: + kfree(hv_domain); + return ERR_PTR(ret); +} + +static struct iommu_ops hv_iommu_ops = { + .capable = hv_iommu_capable, + .domain_alloc_paging = hv_iommu_domain_alloc_paging, + .probe_device = hv_iommu_probe_device, + .device_group = hv_iommu_device_group, + .get_resv_regions = hv_iommu_get_resv_regions, + .owner = THIS_MODULE, + .identity_domain = &hv_identity_domain.domain, + .blocked_domain = &hv_blocking_domain.domain, + .release_domain = &hv_blocking_domain.domain, +}; + +static u64 hv_iommu_detect(struct hv_output_get_iommu_capabilities *cap) +{ + u64 status; + unsigned long flags; + struct hv_input_get_iommu_capabilities *input; + struct hv_output_get_iommu_capabilities *output; + + local_irq_save(flags); + + /* + * The combined input and output fit in the per-CPU input page, + * so no separate output page is needed. + */ + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = (struct hv_output_get_iommu_capabilities *)(input + 1); + memset(input, 0, sizeof(*input)); + input->partition_id = HV_PARTITION_ID_SELF; + status = hv_do_hypercall(HVCALL_GET_IOMMU_CAPABILITIES, input, output); + *cap = *output; + + local_irq_restore(flags); + + return status; +} + +static void __init +hv_init_iommu_device(struct hv_iommu_dev *hv_iommu, + struct hv_output_get_iommu_capabilities *hv_iommu_cap) +{ + ida_init(&hv_iommu->domain_ids); + + hv_iommu->cap = hv_iommu_cap->iommu_cap; + hv_iommu->max_iova_width = hv_iommu_cap->max_iova_width; + if (!hv_iommu_5lvl_supported(hv_iommu->cap) && + hv_iommu->max_iova_width > 48) { + pr_info("5-level paging not supported, limiting iova width to 48.\n"); + hv_iommu->max_iova_width = 48; + } + + hv_iommu->first_domain = HV_DEVICE_DOMAIN_ID_DEFAULT + 1; + hv_iommu->last_domain = HV_DEVICE_DOMAIN_ID_NULL - 1; + hv_iommu->pgsize_bitmap = hv_iommu_cap->pgsize_bitmap; + hv_iommu_device = hv_iommu; +} + +int __init hv_iommu_init(void) +{ + u64 status; + int ret = 0; + struct hv_iommu_dev *hv_iommu = NULL; + struct hv_output_get_iommu_capabilities hv_iommu_cap = {0}; + + if (no_iommu || iommu_detected) + return -ENODEV; + + if (!hv_is_hyperv_initialized()) + return -ENODEV; + + status = hv_iommu_detect(&hv_iommu_cap); + if (!hv_result_success(status)) { + if (hv_result(status) == HV_STATUS_INVALID_HYPERCALL_CODE) + return -ENODEV; + + hv_status_err(status, "HVCALL_GET_IOMMU_CAPABILITIES failed\n"); + return hv_result_to_errno(status); + } + + if (!hv_iommu_present(hv_iommu_cap.iommu_cap)) + return -ENODEV; + + if (!hv_iommu_s1_domain_supported(hv_iommu_cap.iommu_cap)) { + pr_err("stage-1 translation not supported: cap=%#llx\n", + hv_iommu_cap.iommu_cap); + return -ENODEV; + } + + /* + * Require the base page size. The domain page-size bitmap is later + * restricted to the sizes supported by both iommupt and Hyper-V. + */ + if (!(hv_iommu_cap.pgsize_bitmap & PAGE_SIZE)) { + pr_err("unsupported page sizes: pgsize_bitmap=0x%llx\n", + hv_iommu_cap.pgsize_bitmap); + return -ENODEV; + } + + iommu_detected = 1; + pci_request_acs(); + + hv_iommu = kzalloc_obj(*hv_iommu, GFP_KERNEL); + if (!hv_iommu) + return -ENOMEM; + + hv_init_iommu_device(hv_iommu, &hv_iommu_cap); + + ret = hv_initialize_static_domains(); + if (ret) + goto err_free; + + ret = iommu_device_sysfs_add(&hv_iommu->iommu, NULL, NULL, "%s", "hv-iommu"); + if (ret) { + pr_err("iommu_device_sysfs_add failed: %d\n", ret); + goto err_delete_static_domains; + } + + ret = iommu_device_register(&hv_iommu->iommu, &hv_iommu_ops, NULL); + if (ret) { + pr_err("iommu_device_register failed: %d\n", ret); + goto err_sysfs_remove; + } + + pr_info("initialized: %u-bit IOVA aperture, page-size bitmap %#llx\n", + hv_iommu->max_iova_width, hv_iommu->pgsize_bitmap); + return 0; + +err_sysfs_remove: + iommu_device_sysfs_remove(&hv_iommu->iommu); +err_delete_static_domains: + hv_delete_device_domain(&hv_blocking_domain); + hv_delete_device_domain(&hv_identity_domain); +err_free: + kfree(hv_iommu); + return ret; +}
diff --git a/drivers/iommu/hyperv/iommu.h b/drivers/iommu/hyperv/iommu.h new file mode 100644 index 0000000..c412dc1 --- /dev/null +++ b/drivers/iommu/hyperv/iommu.h
@@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Hyper-V IOMMU driver. + * + * Copyright (C) 2024-2026, Microsoft, Inc. + * + */ + +#ifndef _HYPERV_IOMMU_H +#define _HYPERV_IOMMU_H + +struct hv_iommu_dev { + struct iommu_device iommu; + struct ida domain_ids; + + /* Device configuration */ + u8 max_iova_width; + u8 max_pasid_width; + u64 cap; + u64 pgsize_bitmap; + + u64 first_domain; + u64 last_domain; +}; + +struct hv_iommu_domain { + union { + struct iommu_domain domain; + struct pt_iommu pt_iommu; + struct pt_iommu_x86_64 pt_iommu_x86_64; + }; + struct hv_iommu_dev *hv_iommu; + struct hv_input_device_domain device_domain; + u64 pgsize_bitmap; +}; + +PT_IOMMU_CHECK_DOMAIN(struct hv_iommu_domain, pt_iommu, domain); +PT_IOMMU_CHECK_DOMAIN(struct hv_iommu_domain, pt_iommu_x86_64.iommu, domain); + +#define to_hv_iommu_domain(d) \ + container_of(d, struct hv_iommu_domain, domain) + +#endif /* _HYPERV_IOMMU_H */
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index ba675b0..6377ba9 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c
@@ -1344,6 +1344,49 @@ static void qi_dump_fault(struct intel_iommu *iommu, u32 fault) (unsigned long long)desc->qw1); } +static int qi_drain_remaining_descs(struct intel_iommu *iommu, int head, + int wait_index) +{ + struct q_inval *qi = iommu->qi; + int shift = qi_shift(iommu); + struct qi_desc wait_desc = {}; + struct qi_desc nop_desc = {}; + cycles_t start; + + nop_desc.qw0 = QI_IWD_FENCE | QI_IWD_TYPE; + + wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) | QI_IWD_PRQ_DRAIN | + QI_IWD_STATUS_WRITE | QI_IWD_FENCE | QI_IWD_TYPE; + wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]); + + while (head != wait_index) { + memcpy(qi->desc + (head << shift), &nop_desc, 1 << shift); + head = (head + 1) % QI_LENGTH; + } + + WRITE_ONCE(qi->desc_status[wait_index], QI_IN_USE); + memcpy(qi->desc + (wait_index << shift), &wait_desc, 1 << shift); + + /* + * Order the descriptor rewrites before the writel() that restarts + * the fetch engine. + */ + wmb(); + + writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); + + start = get_cycles(); + while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) { + if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start)) { + pr_err("Timeout draining invalidation queue after IQE\n"); + return -ETIMEDOUT; + } + cpu_relax(); + } + + return 0; +} + static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) { u32 fault; @@ -1366,18 +1409,25 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) * is cleared. */ if (fault & DMA_FSTS_IQE) { - head = readl(iommu->reg + DMAR_IQH_REG); - if ((head >> shift) == index) { - struct qi_desc *desc = qi->desc + head; + int head_idx, ret; - /* - * desc->qw2 and desc->qw3 are either reserved or - * used by software as private data. We won't print - * out these two qw's for security consideration. - */ - memcpy(desc, qi->desc + (wait_index << shift), - 1 << shift); - writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); + head = readl(iommu->reg + DMAR_IQH_REG); + head_idx = (head >> shift) % QI_LENGTH; + + /* + * The faulting descriptor can be anywhere within the current + * submission's range [index, wait_index]. Since the queue is + * circular, this submission may wrap around QI_LENGTH + * (index > wait_index in that case), so check both the + * non-wrapped and wrapped cases of the range. + */ + if (index <= wait_index ? + (head_idx >= index && head_idx <= wait_index) : + (head_idx >= index || head_idx <= wait_index)) { + ret = qi_drain_remaining_descs(iommu, head_idx, wait_index); + if (ret) + return ret; + pr_info("Invalidation Queue Error (IQE) cleared\n"); return -EINVAL; }
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 2e3b3ab..5a2e705 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c
@@ -39,8 +39,6 @@ #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e) -#define IOAPIC_RANGE_START (0xfee00000) -#define IOAPIC_RANGE_END (0xfeefffff) #define IOVA_START_ADDR (0x1000) #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57 @@ -1452,12 +1450,103 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu) } } +/* + * Reserve a domain ID inherited from the previous kernel so that it is not + * handed out again while the copied translation structures are still live. + * + * Returns 0 when the ID is reserved, was already reserved, or cannot be + * re-assigned, and a negative errno for a genuine allocation failure. + */ +static int reserve_domain_id(struct intel_iommu *iommu, int did) +{ + int ret; + + if (did < 0 || did >= iommu->max_domain_id) + return 0; + + ret = ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL); + /* + * Devices sharing a domain share its ID, so the same ID is seen in + * more than one context entry; -ENOSPC merely reports that it is + * already reserved. On success the allocated ID is returned, which + * is not an error either. + */ + if (ret == -ENOSPC || ret >= 0) + return 0; + + return ret; +} + +/* + * Reserve the domain IDs used by a scalable mode context entry copied from + * the previous kernel. + */ +static int copy_pasid_table_dids(struct intel_iommu *iommu, struct context_entry *ce) +{ + struct pasid_dir_entry *dir; + unsigned long dir_size; + phys_addr_t dir_phys; + int ret = 0; + int i, j; + + dir_phys = ce->lo & VTD_PAGE_MASK; + if (!dir_phys) + return 0; + + dir_size = get_pasid_dir_size(ce); + dir = memremap(dir_phys, dir_size * sizeof(*dir), MEMREMAP_WB); + if (!dir) + return -ENOMEM; + + for (i = 0; i < dir_size; i++) { + struct pasid_entry *table; + phys_addr_t table_phys; + + if (!pasid_pde_is_present(&dir[i])) + continue; + + /* + * Do not use get_pasid_table_from_pde(); that returns a + * phys_to_virt() pointer, which is not valid for memory + * owned by the previous kernel. + */ + table_phys = READ_ONCE(dir[i].val) & PDE_PFN_MASK; + if (!table_phys) + continue; + + /* A PASID table is one page: PASID_TBL_ENTRIES * 64 bytes. */ + table = memremap(table_phys, PAGE_SIZE, MEMREMAP_WB); + if (!table) { + ret = -ENOMEM; + goto out; + } + + for (j = 0; j < PASID_TBL_ENTRIES; j++) { + if (!pasid_pte_is_present(&table[j])) + continue; + + ret = reserve_domain_id(iommu, pasid_get_domain_id(&table[j])); + if (ret) { + memunmap(table); + goto out; + } + } + + memunmap(table); + } + +out: + memunmap(dir); + + return ret; +} + static int copy_context_table(struct intel_iommu *iommu, struct root_entry *old_re, struct context_entry **tbl, int bus, bool ext) { - int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did; + int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0; struct context_entry *new_ce = NULL, ce; struct context_entry *old_ce = NULL; struct root_entry re; @@ -1519,10 +1608,20 @@ static int copy_context_table(struct intel_iommu *iommu, if (!context_present(&ce)) continue; - - did = context_domain_id(&ce); - if (did >= 0 && did < iommu->max_domain_id) - ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL); + /* + * The context entry only holds a domain ID in legacy mode. + * In scalable mode the IDs are in the PASID table entries. + */ + if (ext) + ret = copy_pasid_table_dids(iommu, &ce); + else + ret = reserve_domain_id(iommu, context_domain_id(&ce)); + if (ret) { + /* Not yet published through @tbl, so free it here. */ + iommu_free_pages(new_ce); + new_ce = NULL; + goto out_unmap; + } set_context_copied(iommu, bus, devfn); new_ce[idx] = ce; @@ -1591,7 +1690,7 @@ static int copy_translation_tables(struct intel_iommu *iommu) if (ret) { pr_err("%s: Failed to copy context table for bus %d\n", iommu->name, bus); - continue; + goto err_free_ctxt_tbls; } } @@ -1623,11 +1722,27 @@ static int copy_translation_tables(struct intel_iommu *iommu) memunmap(old_rt); return 0; +err_free_ctxt_tbls: + /* + * None of these tables have been linked into iommu->root_entry yet, + * so they are unreachable and must be freed here. + */ + for (bus = 0; bus < ctxt_table_entries; bus++) + iommu_free_pages(ctxt_tbls[bus]); + kfree(ctxt_tbls); out_unmap: memunmap(old_rt); err_free_bitmap: bitmap_free(iommu->copied_tables); iommu->copied_tables = NULL; + + /* + * Only reservations taken from the old context entries can be in the + * ida at this point; no domain has been allocated on this IOMMU yet. + * ida_destroy() empties it and leaves it ready for reuse. + */ + ida_destroy(&iommu->domain_ida); + return ret; } @@ -2782,9 +2897,7 @@ static int blocking_domain_attach_dev(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { - struct device_domain_info *info = dev_iommu_priv_get(dev); - - iopf_for_domain_remove(info->domain ? &info->domain->domain : NULL, dev); + iopf_for_domain_remove(old, dev); device_block_translation(dev); return 0; } @@ -2911,10 +3024,10 @@ static unsigned int compute_vasz_lg2_ss(struct intel_iommu *iommu, *top_level = 4; return min(57, mgaw); } else if (mgaw > 39 && sagaw >= BIT(2)) { - *top_level = 3 + ffs(sagaw >> 3); + *top_level = 2 + ffs(sagaw >> 2); return min(48, mgaw); } else if (mgaw > 30 && sagaw >= BIT(1)) { - *top_level = 2 + ffs(sagaw >> 2); + *top_level = 1 + ffs(sagaw >> 1); return min(39, mgaw); } return 0; @@ -3412,7 +3525,6 @@ static void intel_iommu_get_resv_regions(struct device *device, struct list_head *head) { int prot = DMA_PTE_READ | DMA_PTE_WRITE; - struct iommu_resv_region *reg; struct dmar_rmrr_unit *rmrr; struct device *i_dev; int i; @@ -3448,6 +3560,7 @@ static void intel_iommu_get_resv_regions(struct device *device, #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA if (dev_is_pci(device)) { struct pci_dev *pdev = to_pci_dev(device); + struct iommu_resv_region *reg; if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) { reg = iommu_alloc_resv_region(0, 1UL << 24, prot, @@ -3459,12 +3572,7 @@ static void intel_iommu_get_resv_regions(struct device *device, } #endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */ - reg = iommu_alloc_resv_region(IOAPIC_RANGE_START, - IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1, - 0, IOMMU_RESV_MSI, GFP_KERNEL); - if (!reg) - return; - list_add_tail(®->list, head); + iommu_add_resv_x86_msi_region(head); } static struct iommu_group *intel_iommu_device_group(struct device *dev) @@ -3870,6 +3978,14 @@ static int identity_domain_attach_dev(struct iommu_domain *domain, if (dev_is_real_dma_subdevice(dev)) return 0; + if (sm_supported(iommu)) + ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); + else + ret = device_setup_pass_through(dev); + + if (ret) + return ret; + /* * The identity domain has no iopf_handler, so no IOPF reference is * taken for it. The reference held by the old domain must still be @@ -3877,16 +3993,9 @@ static int identity_domain_attach_dev(struct iommu_domain *domain, * not affect the IOPF reference count. */ iopf_for_domain_remove(old, dev); + info->domain_attached = true; - if (sm_supported(iommu)) - ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); - else - ret = device_setup_pass_through(dev); - - if (!ret) - info->domain_attached = true; - - return ret; + return 0; } static int identity_domain_set_dev_pasid(struct iommu_domain *domain,
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 23dbe6c..5b3d234 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h
@@ -1105,13 +1105,21 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, unsigned int size_order, struct qi_desc *desc) { - unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1); - desc->qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) | QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid); /* + * The widest range the descriptor can express is a full flush, encoded + * by making bit 63 the least significant zero bit of ADDR, that is + * @size_order == 64 - VTD_PAGE_SHIFT. Bound @size_order there so that + * a caller passing something larger cannot produce an out-of-range + * shift below. + */ + if (size_order > 64 - VTD_PAGE_SHIFT) + size_order = 64 - VTD_PAGE_SHIFT; + + /* * If S bit is 0, we only flush a single page. If S bit is set, * The least significant zero bit indicates the invalidation address * range. VT-d spec 6.5.2.6. @@ -1120,7 +1128,8 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, * Max Invs Pending (MIP) is set to 0 for now until we have DIT in * ECAP. */ - if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order)) + if (size_order < 64 - VTD_PAGE_SHIFT && + !IS_ALIGNED(addr, BIT_ULL(VTD_PAGE_SHIFT + size_order))) pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n", addr, size_order); @@ -1136,7 +1145,7 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, desc->qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1, VTD_PAGE_SHIFT); /* Clear size_order bit to indicate size */ - desc->qw1 &= ~mask; + desc->qw1 &= ~BIT_ULL(VTD_PAGE_SHIFT + size_order - 1); /* Set the S bit to indicate flushing more than 1 page */ desc->qw1 |= QI_DEV_EIOTLB_SIZE; }
diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index 36e0e8e..df72724 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c
@@ -59,6 +59,8 @@ static int intel_nested_attach_dev(struct iommu_domain *domain, if (ret) goto disable_iopf; + iopf_for_domain_remove(old, dev); + info->domain = dmar_domain; info->domain_attached = true; spin_lock_irqsave(&dmar_domain->lock, flags);
diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h index aaffad5..1f9f886 100644 --- a/drivers/iommu/iommu-priv.h +++ b/drivers/iommu/iommu-priv.h
@@ -30,6 +30,10 @@ static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwsp void iommu_fwspec_free(struct device *dev); +int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); + +struct iommu_domain *iommu_get_dma_domain(struct device *dev); + int iommu_device_register_bus(struct iommu_device *iommu, const struct iommu_ops *ops, const struct bus_type *bus,
diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 9742cb5..9827cb8 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c
@@ -11,7 +11,6 @@ #include "iommu-priv.h" static DEFINE_MUTEX(iommu_sva_lock); -static bool iommu_sva_present; static LIST_HEAD(iommu_sva_mms); static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm); @@ -137,11 +136,9 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm goto out_free_domain; domain->users = 1; - if (list_empty(&iommu_mm->sva_domains)) { - if (list_empty(&iommu_sva_mms)) - iommu_sva_present = true; + if (list_empty(&iommu_mm->sva_domains)) list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms); - } + list_add(&domain->next, &iommu_mm->sva_domains); out: refcount_set(&handle->users, 1); @@ -182,11 +179,8 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) iommu_detach_device_pasid(domain, dev, iommu_mm->pasid); if (--domain->users == 0) { list_del(&domain->next); - if (list_empty(&iommu_mm->sva_domains)) { + if (list_empty(&iommu_mm->sva_domains)) list_del(&iommu_mm->mm_list_elm); - if (list_empty(&iommu_sva_mms)) - iommu_sva_present = false; - } iommu_domain_free(domain); } @@ -334,8 +328,6 @@ void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) struct iommu_mm_data *iommu_mm; guard(mutex)(&iommu_sva_lock); - if (!iommu_sva_present) - return; list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm) mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end);
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index cd1bca7..b486b8b 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c
@@ -4243,6 +4243,10 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) if (!group) return 0; + ret = iommu_deferred_attach(dev, iommu_get_domain_for_dev(dev)); + if (ret) + return ret; + mutex_lock(&group->mutex); /* An IDENTITY domain must pass through */ if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index fe8e6d0..fbdca69 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c
@@ -16,6 +16,7 @@ #include <linux/acpi_rimt.h> #include <linux/compiler.h> #include <linux/crash_dump.h> +#include <linux/dma-map-ops.h> #include <linux/init.h> #include <linux/iommu.h> #include <linux/iopoll.h> @@ -1444,6 +1445,16 @@ static struct iommu_group *riscv_iommu_device_group(struct device *dev) return generic_device_group(dev); } +static bool riscv_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return dev_is_dma_coherent(dev); + default: + return false; + } +} + static int riscv_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); @@ -1505,6 +1516,7 @@ static void riscv_iommu_release_device(struct device *dev) static const struct iommu_ops riscv_iommu_ops = { .of_xlate = riscv_iommu_of_xlate, + .capable = riscv_iommu_capable, .identity_domain = &riscv_iommu_identity_domain, .blocked_domain = &riscv_iommu_blocking_domain, .release_domain = &riscv_iommu_blocking_domain,
diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index fb1cece..9d60f47 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c
@@ -21,8 +21,6 @@ */ #define LPI_ID_BITS_LINEAR 12 -#define IRS_FLAGS_NON_COHERENT BIT(0) - static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); static LIST_HEAD(irs_nodes); @@ -50,6 +48,11 @@ static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, writeq_relaxed(val, irs_data->irs_base + reg_offset); } +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void) +{ + return per_cpu(per_cpu_irs_data, 0); +} + /* * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register * provides the memory barriers (through MMIO accessors) @@ -266,24 +269,24 @@ int gicv5_irs_iste_alloc(const u32 lpi) * itself is not supported) again serves to make it easier to find physically * contiguous blocks of memory. */ -static unsigned int gicv5_irs_l2_sz(u32 idr2) +unsigned int gicv5_irs_l2_sz(u32 l2sz) { switch (PAGE_SIZE) { case SZ_64K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_64K; fallthrough; case SZ_4K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; fallthrough; case SZ_16K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_16K; break; } - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; return GICV5_IRS_IST_CFGR_L2SZ_64K; @@ -331,7 +334,7 @@ static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); if (two_levels) - l2sz = gicv5_irs_l2_sz(idr2); + l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2)); istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index 74be71b..8c0303a 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c
@@ -1122,8 +1122,10 @@ static void gicv5_set_cpuif_idbits(void) #ifdef CONFIG_KVM static struct gic_kvm_info gic_v5_kvm_info __initdata; -static void __init gic_of_setup_kvm_info(struct device_node *node) +static int __init gic_setup_kvm_info(unsigned int maint_irq) { + struct gicv5_irs_chip_data *irs_data = gicv5_irs_get_chip_data(); + /* * If we don't have native GICv5 virtualisation support, then * we also don't have FEAT_GCIE_LEGACY - the architecture @@ -1131,25 +1133,120 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) */ if (!gicv5_global_data.virt_capable) { pr_info("GIC implementation is not virtualization capable\n"); - return; + return -ENODEV; } + if (WARN_ON(!irs_data)) + return -ENODEV; + gic_v5_kvm_info.type = GIC_V5; + gic_v5_kvm_info.gicv5_irs.base = irs_data->irs_base; + gic_v5_kvm_info.gicv5_irs.non_coherent = !!(irs_data->flags & IRS_FLAGS_NON_COHERENT); /* GIC Virtual CPU interface maintenance interrupt */ gic_v5_kvm_info.no_maint_irq_mask = false; - gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); - if (!gic_v5_kvm_info.maint_irq) { - pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n"); + gic_v5_kvm_info.maint_irq = maint_irq; + + /* + * We require an MI if we have legacy support, but don't, otherwise. + * Given that there's an existing flag to convey that an MI isn't + * needed, we (ab)use it to tell KVM that the MI isn't needed if we + * don't support legacy. + * + * The check for ARM64_HAS_GICV5_LEGACY explicitly doesn't use + * cpus_have_final_cap() here as we run too early. + */ + if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && !gic_v5_kvm_info.maint_irq) + gic_v5_kvm_info.no_maint_irq_mask = true; + + vgic_set_kvm_info(&gic_v5_kvm_info); + return 0; +} + +static void __init gic_of_setup_kvm_info(struct device_node *node) +{ + unsigned int maint_irq; + + /* GIC Virtual CPU interface maintenance interrupt */ + maint_irq = irq_of_parse_and_map(node, 0); + if (gic_setup_kvm_info(maint_irq)) + irq_dispose_mapping(maint_irq); +} + +#ifdef CONFIG_ACPI +struct gicv5_acpi_kvm_info { + u32 maint_irq; + bool valid; +}; + +static struct gicv5_acpi_kvm_info acpi_v5_kvm_info __initdata; + +static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_generic_interrupt *gicc = + (struct acpi_madt_generic_interrupt *)header; + + if (!(gicc->flags & + (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) + return 0; + + if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) + pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n"); + + if (!acpi_v5_kvm_info.valid) { + acpi_v5_kvm_info.valid = true; + acpi_v5_kvm_info.maint_irq = gicc->vgic_interrupt; + return 0; + } + + if (acpi_v5_kvm_info.maint_irq != gicc->vgic_interrupt) + pr_warn_once(FW_BUG "MI not the same on all CPUs\n"); + + return 0; +} + +static bool __init gic_acpi_collect_virt_info(void) +{ + acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, + gic_acpi_parse_virt_madt_gicc, 0); + + return acpi_v5_kvm_info.valid; +} + +static void __init gic_acpi_setup_kvm_info(void) +{ + int irq = 0; + + if (!gic_acpi_collect_virt_info()) { + pr_warn("Unable to get hardware information used for virtualization\n"); return; } - vgic_set_kvm_info(&gic_v5_kvm_info); + if (acpi_v5_kvm_info.maint_irq) { + irq = acpi_register_gsi(NULL, acpi_v5_kvm_info.maint_irq, + ACPI_LEVEL_SENSITIVE, + ACPI_ACTIVE_HIGH); + if (irq <= 0) { + pr_warn("Failed to register GSI for GICv5 maintenance IRQ\n"); + irq = 0; + } + } + + if (gic_setup_kvm_info(irq) && irq) + acpi_unregister_gsi(acpi_v5_kvm_info.maint_irq); } +#endif // CONFIG_ACPI #else static inline void __init gic_of_setup_kvm_info(struct device_node *node) { } + +#ifdef CONFIG_ACPI +static inline void __init gic_acpi_setup_kvm_info(void) +{ +} +#endif #endif // CONFIG_KVM static int __init gicv5_init_common(struct fwnode_handle *parent_domain) @@ -1256,6 +1353,8 @@ static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsig acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id, gic_v5_get_gsi_handle); + gic_acpi_setup_kvm_info(); + return 0; out_irs:
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index 1caa87d..9ddbfe9 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -105,7 +105,7 @@ static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env) unsigned int soff; soff = -(off + i) - 1; - stack_entry = &state->stack[soff / BPF_REG_SIZE]; + stack_entry = bpf_stack_slot(state, soff / BPF_REG_SIZE); if (stack_entry->slot_type[soff % BPF_REG_SIZE] == STACK_ZERO) continue;
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 77317e5..dbcfc7d 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c
@@ -3747,7 +3747,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev, */ dev->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_BYTES >> SECTOR_SHIFT, - dma_opt_mapping_size(&pdev->dev) >> 9); + dma_max_opt_mapping_size(&pdev->dev) >> SECTOR_SHIFT); dev->ctrl.max_segments = NVME_MAX_SEGS; dev->ctrl.max_integrity_segments = 1; return dev;
diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 85d826f..824cf27 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c
@@ -641,10 +641,7 @@ static void hv_irq_retarget_interrupt(struct irq_data *data) params->int_entry.source = HV_INTERRUPT_SOURCE_MSI; params->int_entry.msi_entry.address.as_uint32 = int_desc->address & 0xffffffff; params->int_entry.msi_entry.data.as_uint32 = int_desc->data; - params->device_id = (hbus->hdev->dev_instance.b[5] << 24) | - (hbus->hdev->dev_instance.b[4] << 16) | - (hbus->hdev->dev_instance.b[7] << 8) | - (hbus->hdev->dev_instance.b[6] & 0xf8) | + params->device_id = hv_build_logical_dev_id_prefix(hbus->hdev) | PCI_FUNC(pdev->devfn); params->int_target.vector = hv_msi_get_int_vector(data); @@ -3742,6 +3739,7 @@ static int hv_pci_probe(struct hv_device *hdev, struct hv_pcibus_device *hbus; int ret, dom; u16 dom_req; + u32 prefix; char *name; bridge = devm_pci_alloc_host_bridge(&hdev->device, 0); @@ -3884,13 +3882,22 @@ static int hv_pci_probe(struct hv_device *hdev, hbus->state = hv_pcibus_probed; - ret = create_root_hv_pci_bus(hbus); + /* Register the bus before scanning any devices on it. */ + prefix = hv_build_logical_dev_id_prefix(hdev); + + ret = hv_pci_register_dev_id(dom, prefix); if (ret) goto free_windows; + ret = create_root_hv_pci_bus(hbus); + if (ret) + goto unregister_pci_bus; + mutex_unlock(&hbus->state_lock); return 0; +unregister_pci_bus: + hv_pci_unregister_dev_id(dom); free_windows: hv_pci_free_bridge_windows(hbus); exit_d0: @@ -4004,6 +4011,8 @@ static void hv_pci_remove(struct hv_device *hdev) hbus = hv_get_drvdata(hdev); if (hbus->state == hv_pcibus_installed) { + int dom = hbus->bridge->domain_nr; + tasklet_disable(&hdev->channel->callback_event); hbus->state = hv_pcibus_removing; tasklet_enable(&hdev->channel->callback_event); @@ -4021,6 +4030,8 @@ static void hv_pci_remove(struct hv_device *hdev) hv_pci_remove_slots(hbus); pci_remove_root_bus(hbus->bridge->bus); pci_unlock_rescan_remove(); + + hv_pci_unregister_dev_id(dom); } hv_pci_bus_exit(hdev, false);
diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c index 3de05aee..3e58b2c 100644 --- a/drivers/pci/vgaarb.c +++ b/drivers/pci/vgaarb.c
@@ -51,7 +51,8 @@ struct vga_device { unsigned int mem_norm_cnt; /* normal MEM count */ bool bridge_has_one_vga; bool is_firmware_default; /* device selected by firmware */ - unsigned int (*set_decode)(struct pci_dev *pdev, bool decode); + unsigned int (*set_decode)(void *data, bool decode); + void *set_decode_data; }; static LIST_HEAD(vga_list); @@ -935,6 +936,7 @@ EXPORT_SYMBOL(vga_set_legacy_decoding); * vga_client_register - register or unregister a VGA arbitration client * @pdev: PCI device of the VGA client * @set_decode: VGA decode change callback + * @data: private data to pass to callback * * Clients have two callback mechanisms they can use. * @@ -956,15 +958,18 @@ EXPORT_SYMBOL(vga_set_legacy_decoding); * Returns: 0 on success, -ENODEV on failure */ int vga_client_register(struct pci_dev *pdev, - unsigned int (*set_decode)(struct pci_dev *pdev, bool decode)) + unsigned int (*set_decode)(void *data, bool decode), + void *data) { unsigned long flags; struct vga_device *vgadev; spin_lock_irqsave(&vga_lock, flags); vgadev = vgadev_find(pdev); - if (vgadev) + if (vgadev) { vgadev->set_decode = set_decode; + vgadev->set_decode_data = data; + } spin_unlock_irqrestore(&vga_lock, flags); if (!vgadev) return -ENODEV; @@ -1464,7 +1469,7 @@ static void vga_arbiter_notify_clients(void) spin_lock_irqsave(&vga_lock, flags); list_for_each_entry(vgadev, &vga_list, list) { if (vgadev->set_decode) { - new_decodes = vgadev->set_decode(vgadev->pdev, + new_decodes = vgadev->set_decode(vgadev->set_decode_data, new_state); vga_update_device_decodes(vgadev, new_decodes); }
diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 3c64085..664bec2 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
@@ -1135,6 +1135,20 @@ static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev) tm_vdev->status = 0; } +/* Synchronize with callbacks running in the FIFO work item. */ +static void mlxbf_tmfifo_virtio_synchronize_cbs(struct virtio_device *vdev) +{ + struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); + struct mlxbf_tmfifo *fifo = tm_vdev->vrings[0].fifo; + unsigned long flags; + int i; + + for (i = 0; i < ARRAY_SIZE(fifo->spin_lock); i++) { + spin_lock_irqsave(&fifo->spin_lock[i], flags); + spin_unlock_irqrestore(&fifo->spin_lock[i], flags); + } +} + /* Read the value of a configuration field. */ static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev, unsigned int offset, @@ -1179,6 +1193,7 @@ static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = { .find_vqs = mlxbf_tmfifo_virtio_find_vqs, .del_vqs = mlxbf_tmfifo_virtio_del_vqs, .reset = mlxbf_tmfifo_virtio_reset, + .synchronize_cbs = mlxbf_tmfifo_virtio_synchronize_cbs, .set_status = mlxbf_tmfifo_virtio_set_status, .get_status = mlxbf_tmfifo_virtio_get_status, .get = mlxbf_tmfifo_virtio_get,
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 1ed4067..d131b82 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c
@@ -2410,6 +2410,7 @@ static void rproc_type_release(struct device *dev) dev_info(&rproc->dev, "releasing %s\n", rproc->name); + cleanup_srcu_struct(&rproc->vq_srcu); idr_destroy(&rproc->notifyids); if (rproc->index >= 0) @@ -2507,6 +2508,11 @@ struct rproc *rproc_alloc(struct device *dev, const char *name, if (!rproc) return NULL; + if (init_srcu_struct(&rproc->vq_srcu)) { + kfree(rproc); + return NULL; + } + rproc->priv = &rproc[1]; rproc->auto_boot = true; rproc->elf_class = ELFCLASSNONE; @@ -2571,6 +2577,9 @@ EXPORT_SYMBOL(rproc_alloc); * * If no one holds any reference to rproc anymore, then its refcount would * now drop to zero, and it would be freed. + * + * Context: Any context, but the last reference must not be dropped from + * atomic context. */ void rproc_free(struct rproc *rproc) { @@ -2586,6 +2595,9 @@ EXPORT_SYMBOL(rproc_free); * * If no one holds any reference to rproc anymore, then its refcount would * now drop to zero, and it would be freed. + * + * Context: Any context, but the last reference must not be dropped from + * atomic context. */ void rproc_put(struct rproc *rproc) {
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index d5e9ff0..7fefb4b 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -23,6 +23,7 @@ #include <linux/err.h> #include <linux/kref.h> #include <linux/slab.h> +#include <linux/srcu.h> #include "remoteproc_internal.h" @@ -88,15 +89,23 @@ static bool rproc_virtio_notify(struct virtqueue *vq) */ irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int notifyid) { + irqreturn_t ret = IRQ_NONE; struct rproc_vring *rvring; + struct virtqueue *vq; + int idx; dev_dbg(&rproc->dev, "vq index %d is interrupted\n", notifyid); - rvring = idr_find(&rproc->notifyids, notifyid); - if (!rvring || !rvring->vq) - return IRQ_NONE; + idx = srcu_read_lock(&rproc->vq_srcu); - return vring_interrupt(0, rvring->vq); + rvring = idr_find(&rproc->notifyids, notifyid); + vq = rvring ? READ_ONCE(rvring->vq) : NULL; + if (vq) + ret = vring_interrupt(0, vq); + + srcu_read_unlock(&rproc->vq_srcu, idx); + + return ret; } EXPORT_SYMBOL(rproc_vq_interrupt); @@ -153,7 +162,7 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, vq->num_max = num; - rvring->vq = vq; + WRITE_ONCE(rvring->vq, vq); vq->priv = rvring; /* Update vring in resource table */ @@ -165,14 +174,20 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, static void __rproc_virtio_del_vqs(struct virtio_device *vdev) { + struct rproc *rproc = vdev_to_rproc(vdev); struct virtqueue *vq, *n; struct rproc_vring *rvring; - list_for_each_entry_safe(vq, n, &vdev->vqs, list) { + list_for_each_entry(vq, &vdev->vqs, list) { rvring = vq->priv; - rvring->vq = NULL; - vring_del_virtqueue(vq); + WRITE_ONCE(rvring->vq, NULL); } + + /* Synchronize with rproc_vq_interrupt() callers that found a queue. */ + synchronize_srcu(&rproc->vq_srcu); + + list_for_each_entry_safe(vq, n, &vdev->vqs, list) + vring_del_virtqueue(vq); } static void rproc_virtio_del_vqs(struct virtio_device *vdev) @@ -242,6 +257,13 @@ static void rproc_virtio_reset(struct virtio_device *vdev) dev_dbg(&vdev->dev, "reset !\n"); } +static void rproc_virtio_synchronize_cbs(struct virtio_device *vdev) +{ + struct rproc *rproc = vdev_to_rproc(vdev); + + synchronize_srcu(&rproc->vq_srcu); +} + /* provide the vdev features as retrieved from the firmware */ static u64 rproc_virtio_get_features(struct virtio_device *vdev) { @@ -330,6 +352,7 @@ static const struct virtio_config_ops rproc_virtio_config_ops = { .find_vqs = rproc_virtio_find_vqs, .del_vqs = rproc_virtio_del_vqs, .reset = rproc_virtio_reset, + .synchronize_cbs = rproc_virtio_synchronize_cbs, .set_status = rproc_virtio_set_status, .get_status = rproc_virtio_get_status, .get = rproc_virtio_get,
diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 821b617..1ffcd4c 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c
@@ -96,6 +96,7 @@ static void __init sclp_early_facilities_detect(void) sclp.has_ib = cpue->ib; sclp.has_cei = cpue->cei; sclp.has_skey = cpue->skey; + sclp.has_aef = cpue->aef; break; }
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 4db878c..373ce57 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -1822,17 +1822,27 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { /** * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed - * to manage AP resources for the guest whose state is represented by @kvm + * to manage AP resources for the guest whose state is represented by + * @kvm_file * * @matrix_mdev: a mediated matrix device - * @kvm: reference to KVM instance + * @kvm_file: the KVM VM file this vfio device is associated with * - * Return: 0 if no other mediated matrix device has a reference to @kvm; + * Return: 0 if no other mediated matrix device has a reference to the VM; * otherwise, returns an -EPERM. */ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, - struct kvm *kvm) + struct file *kvm_file) { + struct kvm *kvm; + + if (!kvm_file) + return -ENOENT; + + kvm = file_to_kvm_s390(kvm_file); + if (!kvm) + return -ENOENT; + if (kvm->arch.crypto.crycbd) { get_update_locks_for_kvm(kvm); if (kvm->arch.crypto.pqap_hook) { @@ -1841,7 +1851,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, } kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; - kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; vfio_ap_mdev_update_guest_apcb(matrix_mdev); release_update_locks_for_kvm(kvm); @@ -1894,7 +1903,6 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) matrix_mdev->kvm = NULL; release_update_locks_for_kvm(kvm); - kvm_put_kvm(kvm); } }
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index bab6cad..552d779 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c
@@ -1062,7 +1062,7 @@ static void virtio_ccw_synchronize_cbs(struct virtio_device *vdev) struct virtio_ccw_device *vcdev = to_vc_device(vdev); struct airq_info *info = vcdev->airq_info; - if (info) { + if (vcdev->is_thinint && info) { /* * This device uses adapter interrupts: synchronize with * vring_interrupt() called by virtio_airq_handler() @@ -1204,13 +1204,11 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev, vcdev->err = -EIO; } virtio_ccw_check_activity(vcdev, activity); -#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION /* * Paired with virtio_ccw_synchronize_cbs() and interrupts are * disabled here. */ read_lock(&vcdev->irq_lock); -#endif for_each_set_bit(i, indicators(vcdev), sizeof(*indicators(vcdev)) * BITS_PER_BYTE) { /* The bit clear must happen before the vring kick. */ @@ -1219,9 +1217,7 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev, vq = virtio_ccw_vq_by_ind(vcdev, i); vring_interrupt(0, vq); } -#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION read_unlock(&vcdev->irq_lock); -#endif if (test_bit(0, indicators2(vcdev))) { virtio_config_changed(&vcdev->vdev); clear_bit(0, indicators2(vcdev));
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index a825966..50ecce7 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c
@@ -225,8 +225,8 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy) /* * Set shost->opt_sectors from the DMA optimal mapping size, but only - * when dma_opt_mapping_size() is strictly less than dma_max_mapping_size(), - * indicating a genuine optimization hint from an IOMMU or DMA backend. + * when dma_max_opt_mapping_size() is strictly less than + * dma_max_mapping_size(), indicating a genuine optimization hint. * When the two are equal (e.g. IOMMU disabled / passthrough), no real * hint exists, so leave opt_sectors at 0 to avoid bogus optimal_io_size * values that break filesystem geometry (e.g. mkfs.xfs stripe alignment). @@ -234,16 +234,16 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy) static void sas_dma_setup_opt_sectors(struct Scsi_Host *shost) { struct device *dma_dev = shost->dma_dev; - size_t opt = dma_opt_mapping_size(dma_dev); + size_t max_opt = dma_max_opt_mapping_size(dma_dev); size_t max = dma_max_mapping_size(dma_dev); unsigned int opt_sectors; - /* opt >= max means no real hint was provided by the DMA layer */ - if (opt >= max) + /* max_opt >= max means no real hint was provided by the DMA layer */ + if (max_opt >= max) return; /* Clamp to max_sectors to avoid overflow in sector arithmetic */ - opt_sectors = min_t(unsigned int, opt >> SECTOR_SHIFT, + opt_sectors = min_t(unsigned int, max_opt >> SECTOR_SHIFT, shost->max_sectors); /* Guard against zero before rounddown_pow_of_two() */
diff --git a/drivers/vfio/cdx/main.c b/drivers/vfio/cdx/main.c index b31ed4b..28dc8c8 100644 --- a/drivers/vfio/cdx/main.c +++ b/drivers/vfio/cdx/main.c
@@ -254,10 +254,10 @@ static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region, if (base + size > region.size) return -EINVAL; - vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; vma->vm_page_prot = pgprot_device(vma->vm_page_prot); - return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + return io_remap_pfn_range(vma, vma->vm_start, + (region.addr >> PAGE_SHIFT) + pgoff, size, vma->vm_page_prot); } @@ -271,6 +271,9 @@ static int vfio_cdx_mmap(struct vfio_device *core_vdev, index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT); + if (!(vma->vm_flags & VM_SHARED)) + return -EINVAL; + if (index >= cdx_dev->res_count) return -EINVAL; @@ -281,9 +284,13 @@ static int vfio_cdx_mmap(struct vfio_device *core_vdev, (vma->vm_flags & VM_READ)) return -EPERM; - if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) && - (vma->vm_flags & VM_WRITE)) - return -EPERM; + /* Prevent read-only region mappings from being upgraded with mprotect() */ + if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)) { + if (vma->vm_flags & VM_WRITE) + return -EPERM; + + vm_flags_clear(vma, VM_MAYWRITE); + } return vfio_cdx_mmap_mmio(vdev->regions[index], vma); }
diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c index 1d9515c..9262729 100644 --- a/drivers/vfio/device_cdev.c +++ b/drivers/vfio/device_cdev.c
@@ -65,7 +65,7 @@ int vfio_device_fops_cdev_open(struct inode *inode, struct file *filep) static void vfio_df_get_kvm_safe(struct vfio_device_file *df) { spin_lock(&df->kvm_ref_lock); - vfio_device_get_kvm_safe(df->device, df->kvm); + vfio_device_get_kvm(df->device, df->kvm); spin_unlock(&df->kvm_ref_lock); } @@ -130,6 +130,18 @@ long vfio_df_ioctl_bind_iommufd(struct vfio_device_file *df, goto out_unlock; } + /* + * The cdev path allows only a single open. Reject a second open here, + * before the VF token and device->kvm updates below would clobber the + * current opener's state on a bind that cannot complete. Return -EBUSY + * rather than -EINVAL since a delayed release of the prior opener can + * make this transient. + */ + if (device->open_count) { + ret = -EBUSY; + goto out_unlock; + } + ret = vfio_df_check_token(device, &bind); if (ret) goto out_unlock;
diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c index b4c39582..15b05b8c 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c
@@ -369,9 +369,8 @@ static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region, if (!region_cacheable) vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; - - return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + return remap_pfn_range(vma, vma->vm_start, + (region.addr >> PAGE_SHIFT) + pgoff, size, vma->vm_page_prot); } @@ -403,9 +402,13 @@ static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev, && (vma->vm_flags & VM_READ)) return -EINVAL; - if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) - && (vma->vm_flags & VM_WRITE)) - return -EINVAL; + /* Prevent read-only region mappings from being upgraded with mprotect() */ + if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)) { + if (vma->vm_flags & VM_WRITE) + return -EINVAL; + + vm_flags_clear(vma, VM_MAYWRITE); + } vma->vm_private_data = mc_dev;
diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index b2299e5..185f359 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c
@@ -163,7 +163,7 @@ static int vfio_group_ioctl_set_container(struct vfio_group *group, static void vfio_device_group_get_kvm_safe(struct vfio_device *device) { spin_lock(&device->group->kvm_ref_lock); - vfio_device_get_kvm_safe(device, device->group->kvm); + vfio_device_get_kvm(device, device->group->kvm); spin_unlock(&device->group->kvm_ref_lock); } @@ -860,11 +860,20 @@ bool vfio_group_enforced_coherent(struct vfio_group *group) return ret; } -void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm) +void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm) { + struct file *old; + + if (kvm) + get_file(kvm); + spin_lock(&group->kvm_ref_lock); + old = group->kvm; group->kvm = kvm; spin_unlock(&group->kvm_ref_lock); + + if (old) + fput(old); } /**
diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c index 3b4aa06..3433729 100644 --- a/drivers/vfio/mdev/mdev_core.c +++ b/drivers/vfio/mdev/mdev_core.c
@@ -50,7 +50,7 @@ static int mdev_device_remove_cb(struct device *dev, void *data) * @types: Array of supported mdev types * @nr_types: Number of entries in @types * - * Registers the @parent stucture as a parent for mdev types and thus mdev + * Registers the @parent structure as a parent for mdev types and thus mdev * devices. The caller needs to hold a reference on @dev that must not be * released until after the call to mdev_unregister_parent(). *
diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index 86362ec..c82c592 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -16,6 +16,9 @@ #include "hisi_acc_vfio_pci.h" +VFIO_PCI_CORE_DEFINE_CALLBACKS(hisi_acc_vf, struct hisi_acc_vf_core_device, + core_device) + /* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */ static int qm_wait_dev_not_ready(struct hisi_qm *qm) { @@ -588,10 +591,7 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); - - return container_of(core_device, struct hisi_acc_vf_core_device, - core_device); + return pci_get_drvdata(pdev); } /* Check the PF's RAS state and Function INT state */ @@ -1154,14 +1154,16 @@ static void hisi_acc_vf_pci_reset_prepare(struct pci_dev *pdev) { struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); struct hisi_qm *qm = hisi_acc_vdev->pf_qm; - struct device *dev = &qm->pdev->dev; u32 delay = 0; + if (!hisi_acc_vdev->core_device.vdev.mig_ops) + return; + /* All reset requests need to be queued for processing */ while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { msleep(1); if (++delay > QM_RESET_WAIT_TIMEOUT) { - dev_err(dev, "reset prepare failed\n"); + dev_err(&qm->pdev->dev, "reset prepare failed\n"); return; } } @@ -1174,12 +1176,14 @@ static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); struct hisi_qm *qm = hisi_acc_vdev->pf_qm; - if (hisi_acc_vdev->set_reset_flag) - clear_bit(QM_RESETTING, &qm->misc_ctl); - if (!hisi_acc_vdev->core_device.vdev.mig_ops) return; + if (hisi_acc_vdev->set_reset_flag) { + clear_bit(QM_RESETTING, &qm->misc_ctl); + hisi_acc_vdev->set_reset_flag = false; + } + mutex_lock(&hisi_acc_vdev->state_mutex); hisi_acc_vf_reset(hisi_acc_vdev); mutex_unlock(&hisi_acc_vdev->state_mutex); @@ -1191,13 +1195,6 @@ static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev) struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; struct pci_dev *vf_dev = vdev->pdev; - u32 val; - - val = readl(pf_qm->io_base + QM_MIG_REGION_SEL); - if (pf_qm->ver > QM_HW_V3 && (val & QM_MIG_REGION_EN)) - hisi_acc_vdev->drv_mode = HW_ACC_MIG_PF_CTRL; - else - hisi_acc_vdev->drv_mode = HW_ACC_MIG_VF_CTRL; if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_PF_CTRL) { /* @@ -1416,9 +1413,8 @@ static int hisi_acc_vf_debug_check(struct seq_file *seq, struct vfio_device *vde static int hisi_acc_vf_debug_cmd(struct seq_file *seq, void *data) { struct device *vf_dev = seq->private; - struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev); - struct vfio_device *vdev = &core_device->vdev; - struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); + struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(vf_dev); + struct vfio_device *vdev = &hisi_acc_vdev->core_device.vdev; struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; u64 value; int ret; @@ -1445,9 +1441,8 @@ static int hisi_acc_vf_debug_cmd(struct seq_file *seq, void *data) static int hisi_acc_vf_dev_read(struct seq_file *seq, void *data) { struct device *vf_dev = seq->private; - struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev); - struct vfio_device *vdev = &core_device->vdev; - struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); + struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(vf_dev); + struct vfio_device *vdev = &hisi_acc_vdev->core_device.vdev; size_t vf_data_sz = offsetofend(struct acc_vf_data, padding); struct acc_vf_data *vf_data; int ret; @@ -1492,9 +1487,7 @@ static int hisi_acc_vf_dev_read(struct seq_file *seq, void *data) static int hisi_acc_vf_migf_read(struct seq_file *seq, void *data) { struct device *vf_dev = seq->private; - struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev); - struct vfio_device *vdev = &core_device->vdev; - struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); + struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(vf_dev); size_t vf_data_sz = offsetofend(struct acc_vf_data, padding); struct hisi_acc_vf_migration_file *debug_migf = hisi_acc_vdev->debug_migf; @@ -1665,21 +1658,34 @@ static void hisi_acc_vf_debugfs_exit(struct hisi_acc_vf_core_device *hisi_acc_vd hisi_acc_vdev->debug_migf = NULL; } +static enum hw_drv_mode hisi_acc_vf_get_drv_mode(struct hisi_qm *pf_qm) +{ + u32 val; + + val = readl(pf_qm->io_base + QM_MIG_REGION_SEL); + if (pf_qm->ver > QM_HW_V3 && (val & QM_MIG_REGION_EN)) + return HW_ACC_MIG_PF_CTRL; + + return HW_ACC_MIG_VF_CTRL; +} + static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct hisi_acc_vf_core_device *hisi_acc_vdev; const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops; + enum hw_drv_mode drv_mode = HW_ACC_MIG_VF_CTRL; + resource_size_t func_len; struct hisi_qm *pf_qm; - int vf_id; int ret; pf_qm = hisi_acc_get_pf_qm(pdev); if (pf_qm && pf_qm->ver >= QM_HW_V3) { - vf_id = pci_iov_vf_id(pdev); - if (vf_id >= 0) - ops = &hisi_acc_vfio_pci_migrn_ops; + func_len = pci_resource_len(pdev, VFIO_PCI_BAR2_REGION_INDEX) >> 1; + drv_mode = hisi_acc_vf_get_drv_mode(pf_qm); + if (drv_mode == HW_ACC_MIG_VF_CTRL && func_len < PAGE_SIZE) + pci_warn(pdev, "migration not supported on 64KB pages with QM_HW_V3\n"); else - pci_warn(pdev, "migration support failed, continue with generic interface\n"); + ops = &hisi_acc_vfio_pci_migrn_ops; } hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device, @@ -1687,7 +1693,9 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device if (IS_ERR(hisi_acc_vdev)) return PTR_ERR(hisi_acc_vdev); - dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device); + hisi_acc_vdev->drv_mode = drv_mode; + + dev_set_drvdata(&pdev->dev, hisi_acc_vdev); ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device); if (ret) goto out_put_vdev; @@ -1721,7 +1729,7 @@ MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); static const struct pci_error_handlers hisi_acc_vf_err_handlers = { .reset_prepare = hisi_acc_vf_pci_reset_prepare, .reset_done = hisi_acc_vf_pci_aer_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = hisi_acc_vf_aer_err_detected, }; static struct pci_driver hisi_acc_vfio_pci_driver = { @@ -1729,6 +1737,7 @@ static struct pci_driver hisi_acc_vfio_pci_driver = { .id_table = hisi_acc_vfio_pci_table, .probe = hisi_acc_vfio_pci_probe, .remove = hisi_acc_vfio_pci_remove, + .driver = { .pm = pm_ptr(&hisi_acc_vf_pm_ops) }, .err_handler = &hisi_acc_vf_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/ism/main.c b/drivers/vfio/pci/ism/main.c index f83e09b..cce3bdf 100644 --- a/drivers/vfio/pci/ism/main.c +++ b/drivers/vfio/pci/ism/main.c
@@ -47,6 +47,9 @@ struct ism_vfio_pci_core_device { struct kmem_cache *store_block_cache; }; +VFIO_PCI_CORE_DEFINE_CALLBACKS(ism_vfio, struct ism_vfio_pci_core_device, + core_device) + static int ism_vfio_pci_open_device(struct vfio_device *core_vdev) { struct ism_vfio_pci_core_device *ivpcd; @@ -363,7 +366,7 @@ static int ism_vfio_pci_probe(struct pci_dev *pdev, if (IS_ERR(ivpcd)) return PTR_ERR(ivpcd); - dev_set_drvdata(&pdev->dev, &ivpcd->core_device); + dev_set_drvdata(&pdev->dev, ivpcd); ret = vfio_pci_core_register_device(&ivpcd->core_device); if (ret) @@ -374,12 +377,7 @@ static int ism_vfio_pci_probe(struct pci_dev *pdev, static void ism_vfio_pci_remove(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device; - struct ism_vfio_pci_core_device *ivpcd; - - core_device = dev_get_drvdata(&pdev->dev); - ivpcd = container_of(core_device, struct ism_vfio_pci_core_device, - core_device); + struct ism_vfio_pci_core_device *ivpcd = pci_get_drvdata(pdev); vfio_pci_core_unregister_device(&ivpcd->core_device); vfio_put_device(&ivpcd->core_device.vdev); @@ -392,12 +390,17 @@ static const struct pci_device_id ism_device_table[] = { }; MODULE_DEVICE_TABLE(pci, ism_device_table); +static const struct pci_error_handlers ism_vfio_err_handlers = { + .error_detected = ism_vfio_aer_err_detected, +}; + static struct pci_driver ism_vfio_pci_driver = { .name = KBUILD_MODNAME, .id_table = ism_device_table, .probe = ism_vfio_pci_probe, .remove = ism_vfio_pci_remove, - .err_handler = &vfio_pci_core_err_handlers, + .driver = { .pm = pm_ptr(&ism_vfio_pm_ops) }, + .err_handler = &ism_vfio_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c index 5fe0621..0418003 100644 --- a/drivers/vfio/pci/mlx5/cmd.c +++ b/drivers/vfio/pci/mlx5/cmd.c
@@ -1644,7 +1644,7 @@ set_report_output(u32 size, int index, struct mlx5_vhca_qp *qp, int i; buf_offset = index * qp->max_msg_size; - if (WARN_ON(buf_offset + size >= qp->recv_buf.npages * PAGE_SIZE || + if (WARN_ON(buf_offset + size > qp->recv_buf.npages * PAGE_SIZE || (nent > qp->max_msg_size / entry_size))) return;
diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c index de306de..18a6d22 100644 --- a/drivers/vfio/pci/mlx5/main.c +++ b/drivers/vfio/pci/mlx5/main.c
@@ -21,6 +21,9 @@ #include "cmd.h" +VFIO_PCI_CORE_DEFINE_CALLBACKS(mlx5vf, struct mlx5vf_pci_core_device, + core_device) + /* Device specification max LOAD size */ #define MAX_LOAD_SIZE (BIT_ULL(__mlx5_bit_sz(load_vhca_state_in, size)) - 1) @@ -28,10 +31,7 @@ static struct mlx5vf_pci_core_device *mlx5vf_drvdata(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); - - return container_of(core_device, struct mlx5vf_pci_core_device, - core_device); + return pci_get_drvdata(pdev); } static void mlx5vf_disable_fd(struct mlx5_vf_migration_file *migf) @@ -1415,7 +1415,7 @@ static int mlx5vf_pci_probe(struct pci_dev *pdev, if (IS_ERR(mvdev)) return PTR_ERR(mvdev); - dev_set_drvdata(&pdev->dev, &mvdev->core_device); + dev_set_drvdata(&pdev->dev, mvdev); ret = vfio_pci_core_register_device(&mvdev->core_device); if (ret) goto out_put_vdev; @@ -1443,7 +1443,7 @@ MODULE_DEVICE_TABLE(pci, mlx5vf_pci_table); static const struct pci_error_handlers mlx5vf_err_handlers = { .reset_done = mlx5vf_pci_aer_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = mlx5vf_aer_err_detected, }; static struct pci_driver mlx5vf_pci_driver = { @@ -1451,6 +1451,7 @@ static struct pci_driver mlx5vf_pci_driver = { .id_table = mlx5vf_pci_table, .probe = mlx5vf_pci_probe, .remove = mlx5vf_pci_remove, + .driver = { .pm = pm_ptr(&mlx5vf_pm_ops) }, .err_handler = &mlx5vf_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index 963fd8d..5ddec34 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c
@@ -73,6 +73,9 @@ struct nvgrace_gpu_pci_core_device { int cxl_dvsec; }; +VFIO_PCI_CORE_DEFINE_CALLBACKS(nvgrace_gpu, struct nvgrace_gpu_pci_core_device, + core_device) + static void nvgrace_gpu_init_fake_bar_emu_regs(struct vfio_device *core_vdev) { struct nvgrace_gpu_pci_core_device *nvdev = @@ -185,10 +188,8 @@ static int nvgrace_gpu_open_device(struct vfio_device *core_vdev) if (ret) return ret; - if (nvdev->usemem.memlength) { - nvgrace_gpu_init_fake_bar_emu_regs(core_vdev); - mutex_init(&nvdev->remap_lock); - } + nvgrace_gpu_init_fake_bar_emu_regs(core_vdev); + mutex_init(&nvdev->remap_lock); /* * GPU readiness is checked by reading the BAR0 registers. @@ -1083,6 +1084,21 @@ static const struct vfio_device_ops nvgrace_gpu_pci_ops = { .detach_ioas = vfio_iommufd_physical_detach_ioas, }; +static int nvgrace_gpu_core_open_device(struct vfio_device *core_vdev) +{ + struct vfio_pci_core_device *vdev = + container_of(core_vdev, struct vfio_pci_core_device, vdev); + int ret; + + ret = vfio_pci_core_enable(vdev); + if (ret) + return ret; + + vfio_pci_core_finish_enable(vdev); + + return 0; +} + static const struct vfio_pci_device_ops nvgrace_gpu_pci_dev_core_ops = { .get_dmabuf_phys = vfio_pci_core_get_dmabuf_phys, }; @@ -1091,7 +1107,7 @@ static const struct vfio_device_ops nvgrace_gpu_pci_core_ops = { .name = "nvgrace-gpu-vfio-pci-core", .init = vfio_pci_core_init_dev, .release = vfio_pci_core_release_dev, - .open_device = nvgrace_gpu_open_device, + .open_device = nvgrace_gpu_core_open_device, .close_device = vfio_pci_core_close_device, .ioctl = vfio_pci_core_ioctl, .get_region_info_caps = vfio_pci_ioctl_get_region_info, @@ -1341,7 +1357,7 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, if (ret) goto out_put_vdev; - dev_set_drvdata(&pdev->dev, &nvdev->core_device); + dev_set_drvdata(&pdev->dev, nvdev); if (ops == &nvgrace_gpu_pci_ops) { nvdev->has_mig_hw_bug = nvgrace_gpu_has_mig_hw_bug(pdev); @@ -1372,10 +1388,10 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, static void nvgrace_gpu_remove(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); + struct nvgrace_gpu_pci_core_device *nvdev = pci_get_drvdata(pdev); - vfio_pci_core_unregister_device(core_device); - vfio_put_device(&core_device->vdev); + vfio_pci_core_unregister_device(&nvdev->core_device); + vfio_put_device(&nvdev->core_device.vdev); } static const struct pci_device_id nvgrace_gpu_vfio_pci_table[] = { @@ -1407,17 +1423,14 @@ MODULE_DEVICE_TABLE(pci, nvgrace_gpu_vfio_pci_table); */ static void nvgrace_gpu_vfio_pci_reset_done(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); - struct nvgrace_gpu_pci_core_device *nvdev = - container_of(core_device, struct nvgrace_gpu_pci_core_device, - core_device); + struct nvgrace_gpu_pci_core_device *nvdev = pci_get_drvdata(pdev); nvdev->reset_done = true; } static const struct pci_error_handlers nvgrace_gpu_vfio_pci_err_handlers = { .reset_done = nvgrace_gpu_vfio_pci_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = nvgrace_gpu_aer_err_detected, }; static struct pci_driver nvgrace_gpu_vfio_pci_driver = { @@ -1425,6 +1438,7 @@ static struct pci_driver nvgrace_gpu_vfio_pci_driver = { .id_table = nvgrace_gpu_vfio_pci_table, .probe = nvgrace_gpu_probe, .remove = nvgrace_gpu_remove, + .driver = { .pm = pm_ptr(&nvgrace_gpu_pm_ops) }, .err_handler = &nvgrace_gpu_vfio_pci_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/pds/pci_drv.c b/drivers/vfio/pci/pds/pci_drv.c index 4923f18..cf7e33d 100644 --- a/drivers/vfio/pci/pds/pci_drv.c +++ b/drivers/vfio/pci/pds/pci_drv.c
@@ -16,6 +16,9 @@ #include "pci_drv.h" #include "cmds.h" +VFIO_PCI_CORE_DEFINE_CALLBACKS(pds_vfio, struct pds_vfio_pci_device, + vfio_coredev) + #define PDS_VFIO_DRV_DESCRIPTION "AMD/Pensando VFIO Device Driver" #define PCI_VENDOR_ID_PENSANDO 0x1dd8 @@ -118,7 +121,7 @@ static int pds_vfio_pci_probe(struct pci_dev *pdev, if (IS_ERR(pds_vfio)) return PTR_ERR(pds_vfio); - dev_set_drvdata(&pdev->dev, &pds_vfio->vfio_coredev); + dev_set_drvdata(&pdev->dev, pds_vfio); err = vfio_pci_core_register_device(&pds_vfio->vfio_coredev); if (err) @@ -173,7 +176,7 @@ static void pds_vfio_pci_aer_reset_done(struct pci_dev *pdev) static const struct pci_error_handlers pds_vfio_pci_err_handlers = { .reset_done = pds_vfio_pci_aer_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = pds_vfio_aer_err_detected, }; static struct pci_driver pds_vfio_pci_driver = { @@ -181,6 +184,7 @@ static struct pci_driver pds_vfio_pci_driver = { .id_table = pds_vfio_pci_table, .probe = pds_vfio_pci_probe, .remove = pds_vfio_pci_remove, + .driver = { .pm = pm_ptr(&pds_vfio_pm_ops) }, .err_handler = &pds_vfio_pci_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/pds/vfio_dev.c b/drivers/vfio/pci/pds/vfio_dev.c index be103c7..6d2b2dc6 100644 --- a/drivers/vfio/pci/pds/vfio_dev.c +++ b/drivers/vfio/pci/pds/vfio_dev.c
@@ -20,10 +20,7 @@ struct device *pds_vfio_to_dev(struct pds_vfio_pci_device *pds_vfio) struct pds_vfio_pci_device *pds_vfio_pci_drvdata(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); - - return container_of(core_device, struct pds_vfio_pci_device, - vfio_coredev); + return pci_get_drvdata(pdev); } void pds_vfio_reset(struct pds_vfio_pci_device *pds_vfio,
diff --git a/drivers/vfio/pci/qat/main.c b/drivers/vfio/pci/qat/main.c index 60ff907..69448fa 100644 --- a/drivers/vfio/pci/qat/main.c +++ b/drivers/vfio/pci/qat/main.c
@@ -55,6 +55,9 @@ struct qat_vf_core_device { struct qat_vf_migration_file *saving_migf; }; +VFIO_PCI_CORE_DEFINE_CALLBACKS(qat_vf, struct qat_vf_core_device, + core_device) + static int qat_vf_pci_open_device(struct vfio_device *core_vdev) { struct qat_vf_core_device *qat_vdev = @@ -623,9 +626,7 @@ static const struct vfio_device_ops qat_vf_pci_ops = { static struct qat_vf_core_device *qat_vf_drvdata(struct pci_dev *pdev) { - struct vfio_pci_core_device *core_device = pci_get_drvdata(pdev); - - return container_of(core_device, struct qat_vf_core_device, core_device); + return pci_get_drvdata(pdev); } static void qat_vf_pci_aer_reset_done(struct pci_dev *pdev) @@ -651,7 +652,7 @@ qat_vf_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (IS_ERR(qat_vdev)) return PTR_ERR(qat_vdev); - pci_set_drvdata(pdev, &qat_vdev->core_device); + pci_set_drvdata(pdev, qat_vdev); ret = vfio_pci_core_register_device(&qat_vdev->core_device); if (ret) goto out_put_device; @@ -686,7 +687,7 @@ MODULE_DEVICE_TABLE(pci, qat_vf_vfio_pci_table); static const struct pci_error_handlers qat_vf_err_handlers = { .reset_done = qat_vf_pci_aer_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = qat_vf_aer_err_detected, }; static struct pci_driver qat_vf_vfio_pci_driver = { @@ -694,6 +695,7 @@ static struct pci_driver qat_vf_vfio_pci_driver = { .id_table = qat_vf_vfio_pci_table, .probe = qat_vf_vfio_pci_probe, .remove = qat_vf_vfio_pci_remove, + .driver = { .pm = pm_ptr(&qat_vf_pm_ops) }, .err_handler = &qat_vf_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 830369f..e7c012f 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c
@@ -223,13 +223,44 @@ static const struct pci_device_id vfio_pci_table[] = { MODULE_DEVICE_TABLE(pci, vfio_pci_table); +static int __maybe_unused vfio_pci_pm_suspend(struct device *dev) +{ + struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); + + return vfio_pci_core_runtime_suspend(vdev); +} + +static int __maybe_unused vfio_pci_pm_resume(struct device *dev) +{ + struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); + + return vfio_pci_core_runtime_resume(vdev); +} + +static const struct dev_pm_ops vfio_pci_pm_ops = { + SET_RUNTIME_PM_OPS(vfio_pci_pm_suspend, vfio_pci_pm_resume, NULL) +}; + +static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct vfio_pci_core_device *vdev = pci_get_drvdata(pdev); + + return vfio_pci_core_aer_err_detected(vdev); +} + +static const struct pci_error_handlers vfio_pci_err_handlers = { + .error_detected = vfio_pci_aer_err_detected, +}; + static struct pci_driver vfio_pci_driver = { .name = "vfio-pci", .id_table = vfio_pci_table, .probe = vfio_pci_probe, .remove = vfio_pci_remove, .sriov_configure = vfio_pci_sriov_configure, - .err_handler = &vfio_pci_core_err_handlers, + .err_handler = &vfio_pci_err_handlers, + .driver = { .pm = pm_ptr(&vfio_pci_pm_ops) }, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 6757054..fcbb2da 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -162,9 +162,10 @@ static inline void vfio_pci_core_debugfs_init(struct vfio_pci_core_device *vdev) * has no way to get to it and routing can be disabled externally at the * bridge. */ -static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga) +static unsigned int vfio_pci_set_decode(void *data, bool single_vga) { - struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev); + struct vfio_pci_core_device *vdev = data; + struct pci_dev *pdev = vdev->pdev; struct pci_dev *tmp = NULL; unsigned char max_busnr; unsigned int decodes; @@ -488,10 +489,8 @@ static int vfio_pci_core_pm_exit(struct vfio_pci_core_device *vdev, u32 flags, } #ifdef CONFIG_PM -static int vfio_pci_core_runtime_suspend(struct device *dev) +int vfio_pci_core_runtime_suspend(struct vfio_pci_core_device *vdev) { - struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); - down_write(&vdev->memory_lock); /* * The user can move the device into D3hot state before invoking @@ -516,11 +515,10 @@ static int vfio_pci_core_runtime_suspend(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_core_runtime_suspend); -static int vfio_pci_core_runtime_resume(struct device *dev) +int vfio_pci_core_runtime_resume(struct vfio_pci_core_device *vdev) { - struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); - /* * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit * low power mode. @@ -537,6 +535,7 @@ static int vfio_pci_core_runtime_resume(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_core_runtime_resume); #endif /* CONFIG_PM */ /* @@ -576,18 +575,6 @@ static void vfio_pci_core_map_bars(struct vfio_pci_core_device *vdev) } } -/* - * The pci-driver core runtime PM routines always save the device state - * before going into suspended state. If the device is going into low power - * state with only with runtime PM ops, then no explicit handling is needed - * for the devices which have NoSoftRst-. - */ -static const struct dev_pm_ops vfio_pci_core_pm_ops = { - SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend, - vfio_pci_core_runtime_resume, - NULL) -}; - int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) { struct pci_dev *pdev = vdev->pdev; @@ -2159,10 +2146,10 @@ static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev) if (ret) return ret; - ret = vga_client_register(pdev, vfio_pci_set_decode); + ret = vga_client_register(pdev, vfio_pci_set_decode, vdev); if (ret) return ret; - vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false)); + vga_set_legacy_decoding(pdev, vfio_pci_set_decode(vdev, false)); return 0; } @@ -2221,10 +2208,6 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) struct device *dev = &pdev->dev; int ret; - /* Drivers must set the vfio_pci_core_device to their drvdata */ - if (WARN_ON(vdev != dev_get_drvdata(dev))) - return -EINVAL; - /* Drivers must set a name. Required for sequestering SR-IOV VFs */ if (WARN_ON(!vdev->vdev.ops->name)) return -EINVAL; @@ -2292,7 +2275,6 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) */ vfio_pci_set_power_state(vdev, PCI_D0); - dev->driver->pm = &vfio_pci_core_pm_ops; pm_runtime_allow(dev); if (!vdev->disable_idle_d3) pm_runtime_put(dev); @@ -2333,10 +2315,8 @@ void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) } EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device); -pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, - pci_channel_state_t state) +pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev) { - struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev); struct vfio_pci_eventfd *eventfd; rcu_read_lock(); @@ -2419,11 +2399,6 @@ int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, } EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure); -const struct pci_error_handlers vfio_pci_core_err_handlers = { - .error_detected = vfio_pci_core_aer_err_detected, -}; -EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers); - static bool vfio_dev_in_groups(struct vfio_device *vdev, struct vfio_pci_group_info *groups) { @@ -2675,6 +2650,7 @@ static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set) static void vfio_pci_core_cleanup(void) { vfio_pci_uninit_perm_bits(); + rcu_barrier(); } static int __init vfio_pci_core_init(void)
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 7f14dd4..df88480 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -423,9 +423,6 @@ int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, pos >= vdev->msix_offset + vdev->msix_size)) return -EINVAL; - if (count == 8) - return -EINVAL; - io = vfio_pci_core_get_iomap(vdev, bar); if (IS_ERR(io)) return PTR_ERR(io);
diff --git a/drivers/vfio/pci/virtio/main.c b/drivers/vfio/pci/virtio/main.c index d2e5cbc..123ff51 100644 --- a/drivers/vfio/pci/virtio/main.c +++ b/drivers/vfio/pci/virtio/main.c
@@ -18,6 +18,9 @@ #include "common.h" +VFIO_PCI_CORE_DEFINE_CALLBACKS(virtiovf, struct virtiovf_pci_core_device, + core_device) + static int virtiovf_pci_open_device(struct vfio_device *core_vdev) { struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev, @@ -174,7 +177,7 @@ static int virtiovf_pci_probe(struct pci_dev *pdev, if (sup_lm) virtiovf_set_migratable(virtvdev); - dev_set_drvdata(&pdev->dev, &virtvdev->core_device); + dev_set_drvdata(&pdev->dev, virtvdev); ret = vfio_pci_core_register_device(&virtvdev->core_device); if (ret) goto out; @@ -211,7 +214,7 @@ static void virtiovf_pci_aer_reset_done(struct pci_dev *pdev) static const struct pci_error_handlers virtiovf_err_handlers = { .reset_done = virtiovf_pci_aer_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = virtiovf_aer_err_detected, }; static struct pci_driver virtiovf_pci_driver = { @@ -219,6 +222,7 @@ static struct pci_driver virtiovf_pci_driver = { .id_table = virtiovf_pci_table, .probe = virtiovf_pci_probe, .remove = virtiovf_pci_remove, + .driver = { .pm = pm_ptr(&virtiovf_pm_ops) }, .err_handler = &virtiovf_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c index cbff5af..7dab3e0 100644 --- a/drivers/vfio/pci/xe/main.c +++ b/drivers/vfio/pci/xe/main.c
@@ -38,6 +38,9 @@ struct xe_vfio_pci_core_device { struct xe_vfio_pci_migration_file *migf; }; +VFIO_PCI_CORE_DEFINE_CALLBACKS(xe_vfio, struct xe_vfio_pci_core_device, + core_device) + #define xe_vdev_to_dev(xe_vdev) (&(xe_vdev)->core_device.pdev->dev) static void xe_vfio_pci_disable_file(struct xe_vfio_pci_migration_file *migf) @@ -140,7 +143,7 @@ static void xe_vfio_pci_reset_done(struct pci_dev *pdev) static const struct pci_error_handlers xe_vfio_pci_err_handlers = { .reset_prepare = xe_vfio_pci_reset_prepare, .reset_done = xe_vfio_pci_reset_done, - .error_detected = vfio_pci_core_aer_err_detected, + .error_detected = xe_vfio_aer_err_detected, }; static int xe_vfio_pci_open_device(struct vfio_device *core_vdev) @@ -550,7 +553,7 @@ static int xe_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i if (IS_ERR(xe_vdev)) return PTR_ERR(xe_vdev); - dev_set_drvdata(&pdev->dev, &xe_vdev->core_device); + dev_set_drvdata(&pdev->dev, xe_vdev); ret = vfio_pci_core_register_device(&xe_vdev->core_device); if (ret) { @@ -574,9 +577,12 @@ static void xe_vfio_pci_remove(struct pci_dev *pdev) } static const struct pci_device_id xe_vfio_pci_table[] = { + INTEL_BMG_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_CRI_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_NVLS_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_NVLP_IDS(INTEL_PCI_VFIO_DEVICE), INTEL_PTL_IDS(INTEL_PCI_VFIO_DEVICE), INTEL_WCL_IDS(INTEL_PCI_VFIO_DEVICE), - INTEL_BMG_IDS(INTEL_PCI_VFIO_DEVICE), {} }; MODULE_DEVICE_TABLE(pci, xe_vfio_pci_table); @@ -586,6 +592,7 @@ static struct pci_driver xe_vfio_pci_driver = { .id_table = xe_vfio_pci_table, .probe = xe_vfio_pci_probe, .remove = xe_vfio_pci_remove, + .driver = { .pm = pm_ptr(&xe_vfio_pm_ops) }, .err_handler = &xe_vfio_pci_err_handlers, .driver_managed_dma = true, };
diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c index c72db5a..e8cfa76 100644 --- a/drivers/vfio/platform/vfio_platform_common.c +++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -559,9 +559,8 @@ static int vfio_platform_mmap_mmio(struct vfio_platform_region region, return -EINVAL; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; - - return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + return remap_pfn_range(vma, vma->vm_start, + (region.addr >> PAGE_SHIFT) + pgoff, req_len, vma->vm_page_prot); } @@ -591,9 +590,13 @@ int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma && (vma->vm_flags & VM_READ)) return -EINVAL; - if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) - && (vma->vm_flags & VM_WRITE)) - return -EINVAL; + /* Prevent read-only region mappings from being upgraded with mprotect() */ + if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)) { + if (vma->vm_flags & VM_WRITE) + return -EINVAL; + + vm_flags_clear(vma, VM_MAYWRITE); + } vma->vm_private_data = vdev;
diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 7728bc9..2eeb526 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h
@@ -23,7 +23,7 @@ struct vfio_device_file { u8 access_granted; u32 devid; /* only valid when iommufd is valid */ spinlock_t kvm_ref_lock; /* protect kvm field */ - struct kvm *kvm; + struct file *kvm; struct iommufd_ctx *iommufd; /* protected by struct vfio_device_set::lock */ }; @@ -88,7 +88,7 @@ struct vfio_group { #endif enum vfio_group_type type; struct mutex group_lock; - struct kvm *kvm; + struct file *kvm; struct file *opened_file; struct iommufd_ctx *iommufd; spinlock_t kvm_ref_lock; @@ -107,7 +107,7 @@ void vfio_device_group_unuse_iommu(struct vfio_device *device); void vfio_df_group_close(struct vfio_device_file *df); struct vfio_group *vfio_group_from_file(struct file *file); bool vfio_group_enforced_coherent(struct vfio_group *group); -void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm); +void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm); bool vfio_device_has_container(struct vfio_device *device); int __init vfio_group_init(void); void vfio_group_cleanup(void); @@ -165,7 +165,7 @@ static inline bool vfio_group_enforced_coherent(struct vfio_group *group) return true; } -static inline void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm) +static inline void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm) { } @@ -429,11 +429,11 @@ static inline void vfio_virqfd_exit(void) #endif #if IS_ENABLED(CONFIG_KVM) -void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm); +void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm); void vfio_device_put_kvm(struct vfio_device *device); #else -static inline void vfio_device_get_kvm_safe(struct vfio_device *device, - struct kvm *kvm) +static inline void vfio_device_get_kvm(struct vfio_device *device, + struct file *kvm) { }
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index c8151ba..f7addfb 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c
@@ -94,6 +94,7 @@ struct vfio_dma { bool lock_cap; /* capable(CAP_IPC_LOCK) */ bool vaddr_invalid; bool has_rsvd; /* has 1 or more rsvd pfns */ + bool has_non_rsvd; /* has 1 or more !rsvd pfns */ struct task_struct *task; struct rb_root pfn_list; /* Ex-user pinned pfn list */ unsigned long *bitmap; @@ -791,6 +792,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, out: dma->has_rsvd |= rsvd; + dma->has_non_rsvd |= !rsvd; ret = vfio_lock_acct(dma, lock_acct, false); unpin_out: @@ -821,11 +823,13 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, long unlocked = 0, locked = vpfn_pages(dma, iova, npage); if (dma->has_rsvd) { - unsigned long i; + if (dma->has_non_rsvd) { + unsigned long i; - for (i = 0; i < npage; i++) - if (put_pfn(pfn++, dma->prot)) - unlocked++; + for (i = 0; i < npage; i++) + if (put_pfn(pfn++, dma->prot)) + unlocked++; + } } else { put_valid_unreserved_pfns(pfn, npage, dma->prot); unlocked = npage;
diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 423ead4..4e5d738 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c
@@ -472,36 +472,14 @@ void vfio_unregister_group_dev(struct vfio_device *device) EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); #if IS_ENABLED(CONFIG_KVM) -void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) +void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm) { - void (*pfn)(struct kvm *kvm); - bool (*fn)(struct kvm *kvm); - bool ret; - lockdep_assert_held(&device->dev_set->lock); if (!kvm) return; - pfn = symbol_get(kvm_put_kvm); - if (WARN_ON(!pfn)) - return; - - fn = symbol_get(kvm_get_kvm_safe); - if (WARN_ON(!fn)) { - symbol_put(kvm_put_kvm); - return; - } - - ret = fn(kvm); - symbol_put(kvm_get_kvm_safe); - if (!ret) { - symbol_put(kvm_put_kvm); - return; - } - - device->put_kvm = pfn; - device->kvm = kvm; + device->kvm = get_file(kvm); } void vfio_device_put_kvm(struct vfio_device *device) @@ -511,14 +489,7 @@ void vfio_device_put_kvm(struct vfio_device *device) if (!device->kvm) return; - if (WARN_ON(!device->put_kvm)) - goto clear; - - device->put_kvm(device->kvm); - device->put_kvm = NULL; - symbol_put(kvm_put_kvm); - -clear: + fput(device->kvm); device->kvm = NULL; } #endif @@ -603,13 +574,6 @@ int vfio_df_open(struct vfio_device_file *df) lockdep_assert_held(&device->dev_set->lock); - /* - * Only the group path allows the device to be opened multiple - * times. The device cdev path doesn't have a secure way for it. - */ - if (device->open_count != 0 && !df->group) - return -EINVAL; - device->open_count++; if (device->open_count == 1) { ret = vfio_df_device_first_open(df); @@ -1544,9 +1508,13 @@ bool vfio_file_enforced_coherent(struct file *file) } EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); -static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) +static void vfio_device_file_set_kvm(struct file *file, struct file *kvm) { struct vfio_device_file *df = file->private_data; + struct file *old; + + if (kvm) + get_file(kvm); /* * The kvm is first recorded in the vfio_device_file, and will @@ -1554,28 +1522,32 @@ static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) * iommufd successfully in the vfio device cdev path. */ spin_lock(&df->kvm_ref_lock); + old = df->kvm; df->kvm = kvm; spin_unlock(&df->kvm_ref_lock); + + if (old) + fput(old); } /** * vfio_file_set_kvm - Link a kvm with VFIO drivers - * @file: VFIO group file or VFIO device file - * @kvm: KVM to link + * @vfio_file: VFIO group file or VFIO device file + * @kvm: KVM file to link * * When a VFIO device is first opened the KVM will be available in * device->kvm if one was associated with the file. */ -void vfio_file_set_kvm(struct file *file, struct kvm *kvm) +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) { struct vfio_group *group; - group = vfio_group_from_file(file); + group = vfio_group_from_file(vfio_file); if (group) vfio_group_set_kvm(group, kvm); - if (vfio_device_from_file(file)) - vfio_device_file_set_kvm(file, kvm); + if (vfio_device_from_file(vfio_file)) + vfio_device_file_set_kvm(vfio_file, kvm); } EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 2cc73072..4570a4f 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c
@@ -1452,7 +1452,7 @@ static int vhost_net_release(struct inode *inode, struct file *f) vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_cleanup(&n->dev); + vhost_dev_cleanup(&n->dev, false); vhost_net_vq_reset(n); if (tx_sock) sockfd_put(tx_sock); @@ -1661,7 +1661,7 @@ static long vhost_net_reset_owner(struct vhost_net *n) vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_reset_owner(&n->dev, umem); + vhost_dev_reset_owner(&n->dev, umem, false); vhost_net_vq_reset(n); done: mutex_unlock(&n->dev.mutex);
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 4f8c026..885fac5 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c
@@ -2390,7 +2390,7 @@ static int vhost_scsi_release(struct inode *inode, struct file *f) mutex_unlock(&vs->dev.mutex); vhost_scsi_clear_endpoint(vs, &t); vhost_dev_stop(&vs->dev); - vhost_dev_cleanup(&vs->dev); + vhost_dev_cleanup(&vs->dev, false); kfree(vs->dev.vqs); kvfree(vs->vqs); kfree(vs->old_inflight);
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c index 24514c8..84ed9ea 100644 --- a/drivers/vhost/test.c +++ b/drivers/vhost/test.c
@@ -163,7 +163,7 @@ static int vhost_test_release(struct inode *inode, struct file *f) vhost_test_stop(n, &private); vhost_test_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_cleanup(&n->dev); + vhost_dev_cleanup(&n->dev, false); kfree(n->dev.vqs); kfree(n); return 0; @@ -238,7 +238,7 @@ static long vhost_test_reset_owner(struct vhost_test *n) vhost_test_stop(n, &priv); vhost_test_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_reset_owner(&n->dev, umem); + vhost_dev_reset_owner(&n->dev, umem, false); done: mutex_unlock(&n->dev.mutex); return err;
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index a317867..ff7d907 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c
@@ -948,7 +948,7 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_SET_OWNER: r = vhost_vdpa_bind_mm(v); if (r) - vhost_dev_reset_owner(d, NULL); + vhost_dev_reset_owner(d, NULL, false); break; } out: @@ -1453,7 +1453,7 @@ static void vhost_vdpa_cleanup(struct vhost_vdpa *v) } vhost_vdpa_free_domain(v); - vhost_dev_cleanup(&v->vdev); + vhost_dev_cleanup(&v->vdev, false); kfree(v->vdev.vqs); v->vdev.vqs = NULL; }
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 44cac11..4d5d32e 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c
@@ -731,7 +731,7 @@ static void vhost_worker_destroy(struct vhost_dev *dev, kfree(worker); } -static void vhost_workers_free(struct vhost_dev *dev) +static void vhost_workers_free(struct vhost_dev *dev, bool sync) { struct vhost_worker *worker; unsigned long i; @@ -741,6 +741,21 @@ static void vhost_workers_free(struct vhost_dev *dev) for (i = 0; i < dev->nvqs; i++) rcu_assign_pointer(dev->vqs[i]->worker, NULL); + + /* + * This path can be reached by lockless work queuers. In this case + * vhost_vq_work_queue() reads vq->worker under rcu_read_lock(), so a + * RCU reader that fetched a worker before we cleared the pointers above + * may still be queueing work on it. Wait for those readers to finish, + * then flush so any work they queued runs (clearing VHOST_WORK_QUEUED) + * before the workers are freed. Notably, that is the case for the + * vsock RESET_OWNER path. + */ + if (sync) { + synchronize_rcu(); + vhost_dev_flush(dev); + } + /* * Free the default worker we created and cleanup workers userspace * created but couldn't clean up (it forgot or crashed). @@ -1163,11 +1178,12 @@ struct vhost_iotlb *vhost_dev_reset_owner_prepare(void) EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); /* Caller should have device mutex */ -void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem) +void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem, + bool sync) { int i; - vhost_dev_cleanup(dev); + vhost_dev_cleanup(dev, sync); dev->fork_owner = fork_from_owner_default; dev->umem = umem; @@ -1227,7 +1243,7 @@ void vhost_clear_msg(struct vhost_dev *dev) } EXPORT_SYMBOL_GPL(vhost_clear_msg); -void vhost_dev_cleanup(struct vhost_dev *dev) +void vhost_dev_cleanup(struct vhost_dev *dev, bool sync) { int i; @@ -1251,7 +1267,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev) dev->iotlb = NULL; vhost_clear_msg(dev); wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); - vhost_workers_free(dev); + vhost_workers_free(dev, sync); vhost_detach_mm(dev); } EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 39e6121f..5a684ba 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h
@@ -219,8 +219,9 @@ long vhost_dev_set_owner(struct vhost_dev *dev); bool vhost_dev_has_owner(struct vhost_dev *dev); long vhost_dev_check_owner(struct vhost_dev *); struct vhost_iotlb *vhost_dev_reset_owner_prepare(void); -void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb); -void vhost_dev_cleanup(struct vhost_dev *); +void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb, + bool sync); +void vhost_dev_cleanup(struct vhost_dev *dev, bool sync); void vhost_dev_stop(struct vhost_dev *); long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp);
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index abed1fb..0cb8797 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c
@@ -61,6 +61,7 @@ struct vhost_vsock { u32 guest_cid; bool seqpacket_allow; + bool ever_started; /* set on first SET_RUNNING(1); never cleared */ }; static u32 vhost_transport_get_local_cid(void) @@ -302,17 +303,12 @@ vhost_transport_send_pkt(struct sk_buff *skb, struct net *net) return -ENODEV; } - /* Fast-fail if the guest hasn't enabled the RX vq yet. Queuing the packet - * and making the caller wait is pointless: even if the guest manages to init - * within the timeout, it'll immediately reply with RST, because there's no - * listener on the port yet. - * - * vhost_vq_get_backend() without vq->mutex is acceptable here: locking - * the mutex would be too expensive in this hot path, and we already have - * all the outcomes covered: if the backend becomes NULL right after the check, - * vhost_transport_do_send_pkt() will check it under the mutex anyway. + /* Fast-fail until the guest first enables the device (SET_RUNNING(1)). + * Before that there is no listener, so queuing is pointless. + * 'ever_started' is never cleared, so once we're up we keep queuing + * across later stop / CPR-pause windows. */ - if (unlikely(!data_race(vhost_vq_get_backend(&vsock->vqs[VSOCK_VQ_RX])))) { + if (unlikely(!READ_ONCE(vsock->ever_started))) { rcu_read_unlock(); kfree_skb(skb); return -EHOSTUNREACH; @@ -640,11 +636,23 @@ static int vhost_vsock_start(struct vhost_vsock *vsock) mutex_unlock(&vq->mutex); } + /* Set 'ever_started' flag on the first start; never cleared, so send_pkt + * keeps queuing (instead of fast-failing) on later stop / CPR pauses. + */ + WRITE_ONCE(vsock->ever_started, true); + /* Some packets may have been queued before the device was started, * let's kick the send worker to send them. */ vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); + /* The guest may have added TX buffers while the device was stopped + * (e.g. across VHOST_RESET_OWNER) and their kicks got consumed by + * the NULL-backend window. Re-scan the TX VQ, mirroring the RX + * send-worker kick above. + */ + vhost_poll_queue(&vsock->vqs[VSOCK_VQ_TX].poll); + mutex_unlock(&vsock->dev.mutex); return 0; @@ -664,9 +672,24 @@ static int vhost_vsock_start(struct vhost_vsock *vsock) return ret; } +static void vhost_vsock_drop_backends(struct vhost_vsock *vsock) +{ + struct vhost_virtqueue *vq; + size_t i; + + lockdep_assert_held(&vsock->dev.mutex); + + for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { + vq = &vsock->vqs[i]; + + mutex_lock(&vq->mutex); + vhost_vq_set_backend(vq, NULL); + mutex_unlock(&vq->mutex); + } +} + static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner) { - size_t i; int ret = 0; mutex_lock(&vsock->dev.mutex); @@ -677,14 +700,7 @@ static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner) goto err; } - for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { - struct vhost_virtqueue *vq = &vsock->vqs[i]; - - mutex_lock(&vq->mutex); - vhost_vq_set_backend(vq, NULL); - mutex_unlock(&vq->mutex); - } - + vhost_vsock_drop_backends(vsock); err: mutex_unlock(&vsock->dev.mutex); return ret; @@ -720,6 +736,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file) vsock->guest_cid = 0; /* no CID assigned yet */ vsock->seqpacket_allow = false; + vsock->ever_started = false; atomic_set(&vsock->queued_replies, 0); @@ -807,7 +824,7 @@ static int vhost_vsock_dev_release(struct inode *inode, struct file *file) virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); - vhost_dev_cleanup(&vsock->dev); + vhost_dev_cleanup(&vsock->dev, false); put_net_track(vsock->net, &vsock->ns_tracker); kfree(vsock->dev.vqs); vhost_vsock_free(vsock); @@ -888,6 +905,36 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) return -EFAULT; } +static long vhost_vsock_reset_owner(struct vhost_vsock *vsock) +{ + struct vhost_iotlb *umem; + long err; + + mutex_lock(&vsock->dev.mutex); + err = vhost_dev_check_owner(&vsock->dev); + if (err) + goto done; + umem = vhost_dev_reset_owner_prepare(); + if (!umem) { + err = -ENOMEM; + goto done; + } + vhost_vsock_drop_backends(vsock); + vhost_vsock_flush(vsock); + vhost_dev_stop(&vsock->dev); + + /* + * vsock keeps the CID hashed across RESET_OWNER, so a lockless sender + * can still reach the device while its workers are freed. Pass + * sync=true so vhost_workers_free() fences RCU readers and flushes + * before freeing them. + */ + vhost_dev_reset_owner(&vsock->dev, umem, true); +done: + mutex_unlock(&vsock->dev.mutex); + return err; +} + static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl, unsigned long arg) { @@ -931,6 +978,8 @@ static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl, return -EOPNOTSUPP; vhost_set_backend_features(&vsock->dev, features); return 0; + case VHOST_RESET_OWNER: + return vhost_vsock_reset_owner(vsock); default: mutex_lock(&vsock->dev.mutex); r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index b6c9e92..a8588d7 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c
@@ -264,6 +264,8 @@ void virtio_reset_device(struct virtio_device *dev) #endif dev->config->reset(dev); + /* Flush pending VQ/configuration callbacks. */ + virtio_synchronize_cbs(dev); } EXPORT_SYMBOL_GPL(virtio_reset_device);
diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index d9cbb02..8115aa3 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c
@@ -98,8 +98,6 @@ static void vp_reset(struct virtio_device *vdev) /* Flush out the status write, and flush in device writes, * including MSi-X interrupts, if any. */ vp_legacy_get_status(&vp_dev->ldev); - /* Flush pending VQ/configuration callbacks. */ - vp_synchronize_vectors(vdev); } static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 6d8ae2a..922e2df 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c
@@ -364,6 +364,12 @@ static void vp_modern_avq_cleanup(struct virtio_device *vdev) } } +static void vp_modern_del_vqs(struct virtio_device *vdev) +{ + vp_modern_avq_cleanup(vdev); + vp_del_vqs(vdev); +} + static void vp_transport_features(struct virtio_device *vdev, u64 features) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); @@ -557,11 +563,6 @@ static void vp_reset(struct virtio_device *vdev) */ while (vp_modern_get_status(mdev)) msleep(1); - - vp_modern_avq_cleanup(vdev); - - /* Flush pending VQ/configuration callbacks. */ - vp_synchronize_vectors(vdev); } static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) @@ -1232,7 +1233,7 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { .set_status = vp_set_status, .reset = vp_reset, .find_vqs = vp_modern_find_vqs, - .del_vqs = vp_del_vqs, + .del_vqs = vp_modern_del_vqs, .synchronize_cbs = vp_synchronize_vectors, .get_extended_features = vp_get_features, .finalize_features = vp_finalize_features, @@ -1252,7 +1253,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .set_status = vp_set_status, .reset = vp_reset, .find_vqs = vp_modern_find_vqs, - .del_vqs = vp_del_vqs, + .del_vqs = vp_modern_del_vqs, .synchronize_cbs = vp_synchronize_vectors, .get_extended_features = vp_get_features, .finalize_features = vp_finalize_features,
diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 6bcf456..f9559b6 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c
@@ -12,6 +12,7 @@ #include <linux/device.h> #include <linux/kernel.h> #include <linux/slab.h> +#include <linux/srcu.h> #include <linux/uuid.h> #include <linux/group_cpus.h> #include <linux/virtio.h> @@ -27,6 +28,7 @@ struct virtio_vdpa_device { struct virtio_device vdev; struct vdpa_device *vdpa; + struct srcu_struct cb_srcu; u64 features; }; @@ -114,8 +116,11 @@ static bool virtio_vdpa_notify_with_data(struct virtqueue *vq) static irqreturn_t virtio_vdpa_config_cb(void *private) { struct virtio_vdpa_device *vd_dev = private; + int idx; + idx = srcu_read_lock(&vd_dev->cb_srcu); virtio_config_changed(&vd_dev->vdev); + srcu_read_unlock(&vd_dev->cb_srcu, idx); return IRQ_HANDLED; } @@ -123,8 +128,22 @@ static irqreturn_t virtio_vdpa_config_cb(void *private) static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) { struct virtqueue *vq = private; + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); + irqreturn_t ret; + int idx; - return vring_interrupt(0, vq); + idx = srcu_read_lock(&vd_dev->cb_srcu); + ret = vring_interrupt(0, vq); + srcu_read_unlock(&vd_dev->cb_srcu, idx); + + return ret; +} + +static void virtio_vdpa_synchronize_cbs(struct virtio_device *vdev) +{ + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); + + synchronize_srcu(&vd_dev->cb_srcu); } static struct virtqueue * @@ -440,6 +459,7 @@ static const struct virtio_config_ops virtio_vdpa_config_ops = { .reset = virtio_vdpa_reset, .find_vqs = virtio_vdpa_find_vqs, .del_vqs = virtio_vdpa_del_vqs, + .synchronize_cbs = virtio_vdpa_synchronize_cbs, .get_features = virtio_vdpa_get_features, .finalize_features = virtio_vdpa_finalize_features, .bus_name = virtio_vdpa_bus_name, @@ -454,6 +474,7 @@ static void virtio_vdpa_release_dev(struct device *_d) struct virtio_vdpa_device *vd_dev = container_of(vdev, struct virtio_vdpa_device, vdev); + cleanup_srcu_struct(&vd_dev->cb_srcu); kfree(vd_dev); } @@ -467,6 +488,11 @@ static int virtio_vdpa_probe(struct vdpa_device *vdpa) if (!vd_dev) return -ENOMEM; + if (init_srcu_struct(&vd_dev->cb_srcu)) { + kfree(vd_dev); + return -ENOMEM; + } + vd_dev->vdev.dev.parent = vdpa->map ? &vdpa->dev : vdpa_get_map(vdpa).dma_dev; vd_dev->vdev.dev.release = virtio_vdpa_release_dev; @@ -489,10 +515,12 @@ static int virtio_vdpa_probe(struct vdpa_device *vdpa) return 0; err: - if (reg_dev) + if (reg_dev) { put_device(&vd_dev->vdev.dev); - else + } else { + cleanup_srcu_struct(&vd_dev->cb_srcu); kfree(vd_dev); + } return ret; }
diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index abdfbd8..35ccd98 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c
@@ -11,6 +11,7 @@ #include <linux/fsnotify.h> #include <linux/file.h> #include <linux/kernfs.h> +#include <linux/lsm_hooks.h> #include <linux/mm.h> #include <linux/net.h> #include <linux/xattr.h> @@ -328,6 +329,89 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_ return ret; } +static int bpf_inode_init_xattrs_claimed(const struct xattr *xattrs, int xattr_count) +{ + const size_t suffix_len = sizeof(XATTR_BPF_LSM_SUFFIX) - 1; + int i, claimed = 0; + + for (i = 0; i < xattr_count; i++) { + const char *name = xattrs[i].name; + + if (name && !strncmp(name, XATTR_BPF_LSM_SUFFIX, suffix_len)) + claimed++; + } + return claimed; +} + +/** + * bpf_inode_init_xattr - attach a xattr to an inode that is being created + * @xattrs: xattr array the inode_init_security hook was handed + * @xattr_count__ctx_out: xattr count the inode_init_security hook was handed + * @name__str: name of the xattr + * @value_p: xattr value + * + * Claim one of the slots the BPF LSM reserved in the xattr array, so that + * the xattr is written out as part of the transaction creating the inode. + * + * For security reasons, only *name__str* with prefix "security.bpf." is + * allowed. The slot stores the name without that "security." prefix, which + * the filesystem's initxattrs() callback puts back. + * + * At most BPF_LSM_INODE_INIT_XATTRS xattrs can be attached to one inode, + * matching the number of slots the BPF LSM reserves. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs, + int *xattr_count__ctx_out, + const char *name__str, + const struct bpf_dynptr *value_p) +{ + const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + int *xattr_count = xattr_count__ctx_out; + const char *suffix; + struct xattr *slot; + const void *value; + size_t name_len; + u32 value_len; + char *buf; + + if (!match_security_bpf_prefix(name__str)) + return -EPERM; + if (bpf_inode_init_xattrs_claimed(xattrs, *xattr_count) >= + BPF_LSM_INODE_INIT_XATTRS) + return -ENOSPC; + + suffix = name__str + XATTR_SECURITY_PREFIX_LEN; + name_len = strlen(suffix); + if (name_len <= sizeof(XATTR_BPF_LSM_SUFFIX) - 1 || + name_len > XATTR_NAME_MAX - XATTR_SECURITY_PREFIX_LEN) + return -EINVAL; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data(value_ptr, value_len); + if (!value) + return -EINVAL; + if (value_len > XATTR_SIZE_MAX) + return -E2BIG; + + buf = kmalloc(value_len + name_len + 1, GFP_NOWAIT | __GFP_NOWARN); + if (!buf) + return -ENOMEM; + memcpy(buf, value, value_len); + memcpy(buf + value_len, suffix, name_len + 1); + + slot = lsm_get_xattr_slot(xattrs, xattr_count); + if (!slot) { + kfree(buf); + return -ENOSPC; + } + slot->value = buf; + slot->value_len = value_len; + slot->name = buf + value_len; + return 0; +} + #ifdef CONFIG_CGROUPS /** * bpf_cgroup_read_xattr - read xattr of a cgroup's node in cgroupfs @@ -425,6 +509,7 @@ BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_inode_init_xattr) #ifdef CONFIG_NET BTF_ID_FLAGS(func, bpf_sock_read_xattr, KF_RCU) #endif @@ -436,10 +521,24 @@ BTF_ID(func, bpf_set_dentry_xattr) BTF_ID(func, bpf_remove_dentry_xattr) BTF_SET_END(bpf_fs_kfunc_lsm_only_ids) +BTF_ID_LIST_SINGLE(bpf_inode_init_xattr_ids, func, bpf_inode_init_xattr) + +BTF_SET_START(bpf_inode_init_xattr_hooks) +BTF_ID(func, bpf_lsm_inode_init_security) +BTF_SET_END(bpf_inode_init_xattr_hooks) + static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) { if (!btf_id_set8_contains(&bpf_fs_kfunc_set_ids, kfunc_id)) return 0; + if (kfunc_id == bpf_inode_init_xattr_ids[0]) { + if (prog->type != BPF_PROG_TYPE_LSM || + prog->expected_attach_type != BPF_LSM_MAC || + !btf_id_set_contains(&bpf_inode_init_xattr_hooks, + prog->aux->attach_btf_id)) + return -EACCES; + return 0; + } if (prog->type == BPF_PROG_TYPE_LSM) return 0; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
diff --git a/fs/exec.c b/fs/exec.c index 5d7b0fe..8d2b14e 100644 --- a/fs/exec.c +++ b/fs/exec.c
@@ -1471,9 +1471,12 @@ void bprm_drop_loader(struct linux_binprm *bprm) static void free_bprm(struct linux_binprm *bprm) { - if (bprm->mm) { + struct mm_struct *mm = bprm->mm; + + if (mm) { acct_arg_size(bprm, 0); - mmput(bprm->mm); + bprm->mm = NULL; + mmput(mm); } if (bprm->user_ns) put_user_ns(bprm->user_ns);
diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index bf601d6..34d4c33 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h
@@ -73,6 +73,10 @@ extern enum hv_partition_type hv_curr_partition_type; extern void * __percpu *hyperv_pcpu_input_arg; extern void * __percpu *hyperv_pcpu_output_arg; +int hv_pci_register_dev_id(int pci_domain_nr, u32 logical_dev_id_prefix); +void hv_pci_unregister_dev_id(int pci_domain_nr); +int hv_pci_lookup_dev_id(int pci_domain_nr, u32 *prefix); + u64 hv_do_hypercall(u64 control, void *inputaddr, void *outputaddr); u64 hv_do_fast_hypercall8(u16 control, u64 input8); u64 hv_do_fast_hypercall16(u16 control, u64 input1, u64 input2);
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 7a3e9f0..3eb7329 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h
@@ -675,12 +675,43 @@ /* * .BTF */ +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE +#define BTF_INLINE \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline : AT(ADDR(.BTF.inline) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.BTF.inline, _BTF_inline) \ + } +#else +#define BTF_INLINE +#endif + +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE_MODULE +/* __MODULE_NAME_LEN + SHA-256 digest + u32 BTF size */ +#ifdef CONFIG_64BIT +#define BTF_LINK_SIZE (64 - 8 + 32 + 4) +#else +#define BTF_LINK_SIZE (64 - 4 + 32 + 4) +#endif +#define BTF_INLINE_LINK \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline.link : AT(ADDR(.BTF.inline.link) - LOAD_OFFSET) { \ + __start_BTF_inline_link = .; \ + BYTE(0) \ + . += BTF_LINK_SIZE - 1; \ + __stop_BTF_inline_link = .; \ + } +#else +#define BTF_INLINE_LINK +#endif + #ifdef CONFIG_DEBUG_INFO_BTF #define BTF \ . = ALIGN(PAGE_SIZE); \ .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ BOUNDED_SECTION_BY(.BTF, _BTF) \ } \ + BTF_INLINE \ + BTF_INLINE_LINK \ . = ALIGN(PAGE_SIZE); \ .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \ *(.BTF_ids) \
diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h index 6a4e8b9..966945f 100644 --- a/include/hyperv/hvgdk_mini.h +++ b/include/hyperv/hvgdk_mini.h
@@ -37,6 +37,7 @@ struct hv_u128 { #define HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED 0x20 #define HV_STATUS_INVALID_LP_INDEX 0x41 #define HV_STATUS_INVALID_REGISTER_VALUE 0x50 +#define HV_STATUS_DEVICE_ALREADY_IN_DOMAIN 0x66 #define HV_STATUS_OPERATION_FAILED 0x71 #define HV_STATUS_INSUFFICIENT_ROOT_MEMORY 0x73 #define HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY 0x75 @@ -486,10 +487,17 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */ #define HVCALL_GET_VP_INDEX_FROM_APIC_ID 0x009a #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 +#define HVCALL_CREATE_DEVICE_DOMAIN 0x00b1 +#define HVCALL_ATTACH_DEVICE_DOMAIN 0x00b2 #define HVCALL_SIGNAL_EVENT_DIRECT 0x00c0 #define HVCALL_POST_MESSAGE_DIRECT 0x00c1 #define HVCALL_DISPATCH_VP 0x00c2 +#define HVCALL_DETACH_DEVICE_DOMAIN 0x00c4 +#define HVCALL_DELETE_DEVICE_DOMAIN 0x00c5 #define HVCALL_GET_GPA_PAGES_ACCESS_STATES 0x00c9 +#define HVCALL_CONFIGURE_DEVICE_DOMAIN 0x00ce +#define HVCALL_FLUSH_DEVICE_DOMAIN 0x00d0 +#define HVCALL_FLUSH_DEVICE_DOMAIN_LIST 0x00d1 #define HVCALL_ACQUIRE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d7 #define HVCALL_RELEASE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d8 #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db @@ -502,6 +510,8 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */ #define HVCALL_MMIO_READ 0x0106 #define HVCALL_MMIO_WRITE 0x0107 #define HVCALL_DISABLE_HYP_EX 0x010f +#define HVCALL_GET_IOMMU_CAPABILITIES 0x0125 +#define HVCALL_GET_LOGICAL_DEVICE_PROPERTY 0x0127 #define HVCALL_MAP_STATS_PAGE2 0x0131 /* HV_HYPERCALL_INPUT */
diff --git a/include/hyperv/hvhdk_mini.h b/include/hyperv/hvhdk_mini.h index 035ba20..57b70ef 100644 --- a/include/hyperv/hvhdk_mini.h +++ b/include/hyperv/hvhdk_mini.h
@@ -548,4 +548,159 @@ union hv_device_id { /* HV_DEVICE_ID */ } acpi; } __packed; +/* Device domain types */ +#define HV_DEVICE_DOMAIN_TYPE_S1 1 /* HV_DEVICE_DOMAIN_ID_TYPE_S1 */ + +/* ID for default domain and NULL domain */ +#define HV_DEVICE_DOMAIN_ID_DEFAULT 0 +#define HV_DEVICE_DOMAIN_ID_NULL 0xFFFFFFFFULL + +union hv_device_domain_id { /* HV_DEVICE_DOMAIN_ID */ + u64 as_uint64; + struct { + u32 type: 4; + u32 reserved: 28; + u32 id; + } __packed; +}; + +struct hv_input_device_domain { /* HV_INPUT_DEVICE_DOMAIN */ + u64 partition_id; + union hv_input_vtl owner_vtl; + u8 padding[7]; + union hv_device_domain_id domain_id; +} __packed; + +/* HV_CREATE_DEVICE_DOMAIN_FLAGS */ +union hv_create_device_domain_flags { + u32 as_uint32; + struct { + u32 forward_progress_required: 1; + u32 inherit_owning_vtl: 1; + u32 reserved: 30; + } __packed; +}; + +/* HV_INPUT_CREATE_DEVICE_DOMAIN */ +struct hv_input_create_device_domain { + struct hv_input_device_domain device_domain; + union hv_create_device_domain_flags create_device_domain_flags; + u32 padding; +} __packed; + +/* HV_INPUT_DELETE_DEVICE_DOMAIN */ +struct hv_input_delete_device_domain { + struct hv_input_device_domain device_domain; +} __packed; + +/* HV_INPUT_ATTACH_DEVICE_DOMAIN */ +struct hv_input_attach_device_domain { + struct hv_input_device_domain device_domain; + union hv_device_id device_id; +} __packed; + +/* HV_INPUT_DETACH_DEVICE_DOMAIN */ +struct hv_input_detach_device_domain { + u64 partition_id; + union hv_device_id device_id; +} __packed; + +struct hv_device_domain_settings { /* HV_DEVICE_DOMAIN_SETTINGS */ + struct { + /* + * Enable translations. If not enabled, all transaction bypass + * S1 translations. + */ + u64 translation_enabled: 1; + u64 blocked: 1; + /* + * First stage address translation paging mode: + * 0: 4-level paging (default) + * 1: 5-level paging + */ + u64 first_stage_paging_mode: 1; + u64 reserved: 61; + } flags; + + /* Address of translation table */ + u64 page_table_root; +} __packed; + +/* HV_INPUT_CONFIGURE_DEVICE_DOMAIN */ +struct hv_input_configure_device_domain { + struct hv_input_device_domain device_domain; + struct hv_device_domain_settings settings; +} __packed; + +/* HV_INPUT_GET_IOMMU_CAPABILITIES */ +struct hv_input_get_iommu_capabilities { + u64 partition_id; + u64 reserved; +} __packed; + +/* HV_OUTPUT_GET_IOMMU_CAPABILITIES */ +struct hv_output_get_iommu_capabilities { + u32 size; + u16 reserved; + u8 max_iova_width; + u8 max_pasid_width; + +#define HV_IOMMU_CAP_PRESENT BIT_ULL(0) +#define HV_IOMMU_CAP_S2 BIT_ULL(1) +#define HV_IOMMU_CAP_S1 BIT_ULL(2) +#define HV_IOMMU_CAP_S1_5LVL BIT_ULL(3) +#define HV_IOMMU_CAP_PASID BIT_ULL(4) +#define HV_IOMMU_CAP_ATS BIT_ULL(5) +#define HV_IOMMU_CAP_PRI BIT_ULL(6) + + u64 iommu_cap; + u64 pgsize_bitmap; +} __packed; + +/* HV_LOGICAL_DEVICE_PROPERTY_CODE */ +enum hv_logical_device_property_code { + HV_LOGICAL_DEVICE_PROPERTY_PVIOMMU = 10, +}; + +/* HV_INPUT_GET_LOGICAL_DEVICE_PROPERTY */ +struct hv_input_get_logical_device_property { + u64 partition_id; + u64 logical_device_id; + /* Takes values from enum hv_logical_device_property_code. */ + u32 code; + u32 reserved; +} __packed; + +/* HV_OUTPUT_GET_LOGICAL_DEVICE_PROPERTY */ +struct hv_output_get_logical_device_property { +#define HV_DEVICE_IOMMU_ENABLED BIT_ULL(0) + u64 device_iommu; + u64 reserved; +} __packed; + +/* HV_INPUT_FLUSH_DEVICE_DOMAIN */ +struct hv_input_flush_device_domain { + struct hv_input_device_domain device_domain; + u32 flags; + u32 reserved; +} __packed; + +union hv_iommu_flush_va { /* HV_IOMMU_GVA_RANGE */ + u64 as_uint64; + struct { + u64 page_mask_shift : 6; + u64 reserved : 6; + u64 page_number : 52; + }; +} __packed; + +/* HV_INPUT_FLUSH_DEVICE_DOMAIN_LIST */ +struct hv_input_flush_device_domain_list { + struct hv_input_device_domain device_domain; +#define HV_FLUSH_DEVICE_DOMAIN_LIST_IOMMU_FORMAT BIT(0) + u32 flags; + u32 reserved; + union hv_iommu_flush_va iova_list[]; +} __packed; + #endif /* _HV_HVHDK_MINI_H */
diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index bc6f2fd..80f96ea 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h
@@ -195,4 +195,19 @@ static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) WRITE_ONCE(*ctxt->offset.vm_offset, offset); } +/* + * CVAL to program when the hardware won't apply the timer's offset, so that + * its compare against the raw counter matches the guest's at 'now'. An + * expired timer stays expired; one past the wrap never fires. + */ +static inline u64 timer_apply_offset(u64 cval, u64 offset, u64 now) +{ + u64 hw = cval + offset; + + if (now - offset >= cval) + return 0; + + return hw > now ? hw : U64_MAX; +} + #endif
diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index f86a006..06c2061 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h
@@ -10,6 +10,8 @@ #include <linux/kvm_host.h> #include <uapi/linux/psci.h> +#include <asm/kvm_emulate.h> + #define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1) #define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2) #define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0)
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 086b757..7be81b9 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h
@@ -31,7 +31,12 @@ #define VGIC_MAX_SPI 1019 #define VGIC_MAX_RESERVED 1023 #define VGIC_MIN_LPI 8192 -#define KVM_IRQCHIP_NUM_PINS (1020 - 32) + +/* + * The common IRQ routing table must accommodate all GIC types. GICv2 and + * GICv3 need 988 pins, while GICv5 needs 1024 pins. + */ +#define KVM_IRQCHIP_NUM_PINS 1024 /* * GICv5 supports 128 PPIs, but only the first 64 are architected. We only @@ -40,6 +45,8 @@ * in KVM for now. At a future stage, this can be bumped up to 128, if required. */ #define VGIC_V5_NR_PRIVATE_IRQS 64 +#define VGIC_V5_DEFAULT_NR_SPIS 32 +#define VGIC_V5_MAX_NR_SPIS BIT(10) #define is_v5_type(t, i) (FIELD_GET(GICV5_HWIRQ_TYPE, (i)) == (t)) @@ -132,6 +139,19 @@ enum vgic_type { VGIC_V5, /* Newer, fancier GICv5 */ }; +struct vgic_v5_irs_caps { + /* Base address of the host IRS's CONFIG_FRAME */ + void __iomem *irs_base; + + /* Raw host IRS ID registers */ + u32 idr2; + u32 idr3; + u32 idr4; + + /* Is the IRS coherent with us, or not? */ + bool non_coherent; +}; + /* same for all guests, as depending only on the _host's_ GIC model */ struct vgic_global { /* type of the host GIC */ @@ -159,9 +179,16 @@ struct vgic_global { /* Maintenance IRQ number */ unsigned int maint_irq; - /* maximum number of VCPUs allowed (GICv2 limits us to 8) */ + /* + * Maximum number of VCPUs exposed before userspace has selected a + * VGIC model. Individual VGIC models can impose a lower limit + * (GICv2 limits us to 8). + */ int max_gic_vcpus; + /* Maximum number of VCPUs allowed for a GICv5 VM. */ + int max_gicv5_vcpus; + /* Only needed for the legacy KVM_CREATE_IRQCHIP */ bool can_emulate_gicv2; @@ -182,6 +209,9 @@ struct vgic_global { struct { DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); } vgic_v5_ppi_caps; + + /* GICv5 IRS capabilities */ + struct vgic_v5_irs_caps vgic_v5_irs_caps; }; extern struct vgic_global kvm_vgic_global_state; @@ -217,6 +247,12 @@ struct irq_ops { bool (*get_input_level)(int vintid); /* + * Function pointer to directly update hardware pending state after the + * VGIC shadow pending state has changed. + */ + bool (*set_pending_state)(struct kvm_vcpu *vcpu, struct vgic_irq *irq); + + /* * Function pointer to override the queuing of an IRQ. */ bool (*queue_irq_unlock)(struct kvm *kvm, struct vgic_irq *irq, @@ -287,7 +323,8 @@ enum iodev_type { IODEV_CPUIF, IODEV_DIST, IODEV_REDIST, - IODEV_ITS + IODEV_ITS, + IODEV_GICV5_IRS }; struct vgic_io_device { @@ -345,7 +382,11 @@ struct vgic_redist_region { struct list_head list; }; +#define VGIC_V5_VM_ID_INVAL (-1) + struct vgic_v5_vm { + struct kvm *kvm; + /* * We only expose a subset of PPIs to the guest. This subset is a * combination of the PPIs that are actually implemented and what we @@ -363,6 +404,96 @@ struct vgic_v5_vm { * convenient way to do that). */ DECLARE_BITMAP(vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + + struct fwnode_handle *fwnode; + struct irq_domain *domain; + int vpe_db_base; + u32 vm_id; + bool vmte_allocated; +}; + +/*** GICv5 ***/ +struct vgic_v5_irs { + /* base addresses in guest physical address space: */ + gpa_t vgic_v5_irs_base; + + struct vgic_io_device iodev; + struct kvm_device *dev; + + /* IRS state - used for registers etc */ + struct { + u8 domain; + u8 pa_range; + bool virt; + bool setlpi; + bool mec; + bool mpam; + bool swe; + u16 irs_id; + } idr0; + + struct { + /* PE_CNT is populated from online_vcpus at runtime */ + u8 priority_bits; + } idr1; + + struct { + u8 id_bits; + u8 min_lpi_id_bits; + bool ist_levels; + u8 ist_l2sz; + bool istmd; + u8 istmd_sz; + } idr2; + + struct { + u32 spi_range; + } idr5; + + struct { + u32 spi_irs_range; + } idr6; + + struct { + u32 spi_base; + } idr7; + + struct { + u8 sh; + u8 oc; + u8 ic; + bool ist_ra; + bool ist_wa; + bool vmt_ra; + bool vpet_ra; + bool vmd_ra; + bool vmd_wa; + bool vped_ra; + bool vped_wa; + } cr1; + + struct { + u32 id; + } spi_selr; + + struct { + u32 iaffid; + } pe_selr; + + struct { + u8 lpi_id_bits; + u8 l2sz; + u8 istsz; + bool structure; + } ist_cfgr; + + struct { + bool valid; + u64 addr; + } ist_baser; + + /* A valid userspace-restored BASER still needs its host LPI IST. */ + bool lpi_ist_restore_pending; }; struct vgic_dist { @@ -442,6 +573,11 @@ struct vgic_dist { * GICv5 per-VM data. */ struct vgic_v5_vm gicv5_vm; + + /* + * GICv5 IRS data. Dynamically allocated due to the size. + */ + struct vgic_v5_irs *vgic_v5_irs_data; }; struct vgic_v2_cpu_if { @@ -492,6 +628,9 @@ struct vgic_v5_cpu_if { */ u64 vgic_icsr; + /* The contextr used to make VPEs resident and non-resident */ + u64 vgic_contextr; + struct gicv5_vpe gicv5_vpe; }; @@ -598,10 +737,11 @@ void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu); void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1); /** - * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW + * kvm_vgic_get_max_vcpus - Get the pre-VGIC-selection VCPU limit * - * The host's GIC naturally limits the maximum amount of VCPUs a guest - * can use. + * Userspace can query KVM_CAP_MAX_VCPUS before selecting a VGIC model, so + * expose the highest model-specific limit and let kvm_vgic_create() enforce + * the selected model's actual limit. */ static inline int kvm_vgic_get_max_vcpus(void) { @@ -627,7 +767,66 @@ int vgic_v4_load(struct kvm_vcpu *vcpu); void vgic_v4_commit(struct kvm_vcpu *vcpu); int vgic_v4_put(struct kvm_vcpu *vcpu); -int vgic_v5_finalize_ppi_state(struct kvm *kvm); +static inline u8 vgic_v5_irs_ist_id_bits(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_ID_BITS, caps->idr2); +} + +static inline u8 vgic_v5_irs_min_lpi_id_bits(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, caps->idr2); +} + +static inline bool vgic_v5_irs_ist_levels(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr2 & GICV5_IRS_IDR2_IST_LEVELS); +} + +static inline u8 vgic_v5_irs_ist_l2sz(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, caps->idr2); +} + +static inline bool vgic_v5_irs_istmd(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr2 & GICV5_IRS_IDR2_ISTMD); +} + +static inline u8 vgic_v5_irs_istmd_sz(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, caps->idr2); +} + +static inline u32 vgic_v5_irs_max_vms(const struct vgic_v5_irs_caps *caps) +{ + return BIT(FIELD_GET(GICV5_IRS_IDR3_VM_ID_BITS, caps->idr3)); +} + +static inline bool vgic_v5_irs_two_level_vmt_support(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr3 & GICV5_IRS_IDR3_VMT_LEVELS); +} + +static inline u16 vgic_v5_irs_vmd_size(const struct vgic_v5_irs_caps *caps) +{ + if (!(caps->idr3 & GICV5_IRS_IDR3_VMD)) + return 0; + + return BIT(FIELD_GET(GICV5_IRS_IDR3_VMD_SZ, caps->idr3)); +} + +static inline u16 vgic_v5_irs_vped_size(const struct vgic_v5_irs_caps *caps) +{ + return BIT(FIELD_GET(GICV5_IRS_IDR4_VPED_SZ, caps->idr4)); +} + +static inline u32 vgic_v5_irs_max_vpes(const struct vgic_v5_irs_caps *caps) +{ + /* Field stores VPE_ID_BITS - 1 */ + return BIT(FIELD_GET(GICV5_IRS_IDR4_VPE_ID_BITS, caps->idr4) + 1); +} + +int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu); bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, unsigned long flags); void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi);
diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index edcee9f..104817e 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h
@@ -11,11 +11,11 @@ #include <linux/types.h> struct amd_iommu; +struct pci_dev; #ifdef CONFIG_AMD_IOMMU struct task_struct; -struct pci_dev; extern void amd_iommu_detect(void); @@ -76,4 +76,15 @@ static inline int amd_iommu_snp_disable(void) { return 0; } static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif +#ifdef CONFIG_AMD_IOMMU +int amd_iommu_enable_perfopt(struct pci_dev *pdev); +void amd_iommu_disable_perfopt(struct pci_dev *pdev); +#else +static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + return 0; +} +static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { } +#endif + #endif /* _ASM_X86_AMD_IOMMU_H */
diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h index deda507..260a6f4 100644 --- a/include/linux/bootconfig.h +++ b/include/linux/bootconfig.h
@@ -27,6 +27,10 @@ bool __init cmdline_has_extra_options(void); #define BOOTCONFIG_ALIGN (1 << BOOTCONFIG_ALIGN_SHIFT) #define BOOTCONFIG_ALIGN_MASK (BOOTCONFIG_ALIGN - 1) +/* Bootconfig footer is [size][csum][BOOTCONFIG_MAGIC]. */ +#define BOOTCONFIG_FOOTER_SIZE \ + (sizeof(uint32_t) * 2 + BOOTCONFIG_MAGIC_LEN) + /** * xbc_calc_checksum() - Calculate checksum of bootconfig * @data: Bootconfig data. @@ -266,7 +270,15 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node *node, char *buf, size_t size); /* Render key/value pairs under @root as a flat cmdline string */ -int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root); +typedef bool (*xbc_cmdline_filter_fn)(struct xbc_node *node, const char *key, void *data); + +int __init xbc_snprint_cmdline_filter(char *buf, size_t size, struct xbc_node *root, + xbc_cmdline_filter_fn filter, void *data); + +static inline int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) +{ + return xbc_snprint_cmdline_filter(buf, size, root, NULL, NULL); +} /** * xbc_node_compose_key() - Compose full key string of the XBC node @@ -291,12 +303,7 @@ int __init xbc_init(const char *buf, size_t size, const char **emsg, int *epos); int __init xbc_get_info(int *node_size, size_t *data_size); /* XBC cleanup data structures */ -void __init _xbc_exit(bool early); - -static __always_inline void xbc_exit(void) -{ - _xbc_exit(false); -} +void __init xbc_exit(void); /* XBC embedded bootconfig data in kernel */ #ifdef CONFIG_BOOT_CONFIG_EMBED
diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index c9e6b26..0147b8b 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h
@@ -47,6 +47,7 @@ enum cgroup_bpf_attach_type { CGROUP_INET6_GETSOCKNAME, CGROUP_UNIX_GETSOCKNAME, CGROUP_INET_SOCK_RELEASE, + CGROUP_TCP_SOCK_OPS, CGROUP_LSM_START, CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1, MAX_CGROUP_BPF_ATTACH_TYPE
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 4d0cc65..02ef7f8 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h
@@ -100,6 +100,8 @@ struct bpf_cgroup_storage { struct bpf_cgroup_link { struct bpf_link link; struct cgroup *cgroup; + struct bpf_map *map; + wait_queue_head_t wait_hup; }; struct bpf_prog_list { @@ -110,6 +112,18 @@ struct bpf_prog_list { u32 flags; }; +#define bpf_cgroup_struct_ops_foreach(var, item, cgrp, atype) \ + for (item = rcu_dereference((cgrp)->bpf.effective[atype])->items;\ + ((var) = READ_ONCE(item->kdata)); \ + item++) + +static inline bool cgroup_bpf_is_struct_ops_atype(enum cgroup_bpf_attach_type atype) +{ + return atype == CGROUP_TCP_SOCK_OPS; +} +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_trace); +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr); + void __init cgroup_bpf_lifetime_notifier_init(void); int __cgroup_bpf_run_filter_skb(struct sock *sk, @@ -331,27 +345,19 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, * calling bpf_setsockopt on listener-sk will not make sense anyway, * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here. */ -#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - __ret; \ -}) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ + __cgroup_bpf_run_filter_sock_ops(sk, sock_ops, \ + CGROUP_SOCK_OPS) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ - typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ - if (__sk && sk_fullsock(__sk)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - } \ - __ret; \ +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ +({ \ + int __ret = 0; \ + typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ + if (__sk && sk_fullsock(__sk)) \ + __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ + sock_ops, \ + CGROUP_SOCK_OPS); \ + __ret; \ }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) \ @@ -479,6 +485,20 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, return 0; } +static inline bool cgroup_bpf_is_struct_ops_atype(int atype) +{ + return false; +} +static inline void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, + bool mult_trace) +{ +} +static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map, + const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + #define cgroup_bpf_enabled(atype) (0) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; }) #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; }) @@ -501,7 +521,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; }) #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; }) #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0ecb941..580f89c 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h
@@ -17,6 +17,7 @@ #include <linux/numa.h> #include <linux/mm_types.h> #include <linux/wait.h> +#include <linux/irq_work_types.h> #include <linux/refcount.h> #include <linux/mutex.h> #include <linux/module.h> @@ -90,6 +91,7 @@ struct bpf_map_ops { struct bpf_map *(*map_alloc)(union bpf_attr *attr); void (*map_release)(struct bpf_map *map, struct file *map_file); void (*map_free)(struct bpf_map *map); + void (*map_free_pre_rcu)(struct bpf_map *map); int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); void (*map_release_uref)(struct bpf_map *map); void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); @@ -214,6 +216,7 @@ enum btf_field_type { BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), + BPF_RCU_HEAD = (1 << 14), }; enum bpf_cgroup_storage_type { @@ -268,6 +271,7 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; + int rcu_head_off; struct btf_field fields[]; }; @@ -337,12 +341,23 @@ struct bpf_map { bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; bool free_after_rcu_gp; + bool was_inner_map; atomic64_t sleepable_refcnt; s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; + /* + * Which programs use the map, see bpf_map_claim(): 0 - none so far, + * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED + * set - only that one and it stored the addresses of its functions into + * the map, BPF_MAP_USER_MANY - more than one. + */ + unsigned long user; }; +#define BPF_MAP_USER_MANY 1UL +#define BPF_MAP_USER_PATCHED 1UL + static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { @@ -373,6 +388,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; + case BPF_RCU_HEAD: + return "bpf_rcu_head"; default: WARN_ON_ONCE(1); return "unknown"; @@ -413,6 +430,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type) return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); + case BPF_RCU_HEAD: + return sizeof(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -447,6 +466,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type) return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); + case BPF_RCU_HEAD: + return __alignof__(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -479,6 +500,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -572,7 +594,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +602,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) @@ -780,7 +802,10 @@ enum bpf_type_flag { */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), - /* MEM can be uninitialized. */ + /* + * MEM can be uninitialized. Generic memory outputs need not be fully + * initialized by the callee. + */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ @@ -874,7 +899,7 @@ enum bpf_type_flag { /* function argument constraints */ enum bpf_arg_type { - ARG_DONTCARE = 0, /* unused argument in helper function */ + ARG_UNUSED = 0, /* unused argument; terminates argument iteration */ /* the following constraints used to prototype * bpf_map_lookup/update/delete_elem() functions @@ -909,6 +934,24 @@ enum bpf_arg_type { ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */ ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ + + ARG_CONST_SCALAR, /* scalar known at verification time */ + ARG_CONST_MEM_SIZE, /* ARG_MEM_SIZE that must be constant */ + ARG_PTR_TO_ALLOC_BTF_ID, /* pointer to an allocated object */ + ARG_PTR_TO_REFCOUNTED_KPTR, /* pointer to a refcounted local kptr */ + ARG_PTR_TO_ITER, /* pointer to an iterator */ + ARG_PTR_TO_LIST_HEAD, /* pointer to bpf_list_head */ + ARG_PTR_TO_LIST_NODE, /* pointer to bpf_list_node */ + ARG_PTR_TO_RB_ROOT, /* pointer to bpf_rb_root */ + ARG_PTR_TO_RB_NODE, /* pointer to bpf_rb_node */ + ARG_PTR_TO_WORKQUEUE, /* pointer to bpf_wq */ + ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ + ARG_PTR_TO_RCU_HEAD, /* pointer to bpf_rcu_head */ + ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ + ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_PTR_TO_CTX_OUT, /* hook output argument passed through from ctx */ + ARG_PTR_TO_PROG_AUX, /* pointer to the caller's bpf_prog_aux */ + ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ @@ -977,6 +1020,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* A by-value argument takes two eightbytes at most, so the maximum number of + * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may + * need that size for processing, although eventually the maximum slots will + * be capped at MAX_BPF_FUNC_ARGS. + */ +#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS) + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -1005,13 +1055,13 @@ struct bpf_func_proto { }; union { struct { - u32 *arg1_btf_id; - u32 *arg2_btf_id; - u32 *arg3_btf_id; - u32 *arg4_btf_id; - u32 *arg5_btf_id; + const u32 *arg1_btf_id; + const u32 *arg2_btf_id; + const u32 *arg3_btf_id; + const u32 *arg4_btf_id; + const u32 *arg5_btf_id; }; - u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; + const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1114,6 +1164,7 @@ struct bpf_insn_access_aux { u32 ref_id; }; }; + u32 mem_size; struct bpf_verifier_log *log; /* for verbose logs */ bool is_retval; /* is accessing function return value ? */ }; @@ -1194,6 +1245,9 @@ struct bpf_prog_offload { u32 jited_len; }; +/* The argument is aligned to 16 bytes. */ +#define BTF_FMODEL_ALIGN16_ARG BIT(0) + /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) @@ -1211,6 +1265,11 @@ struct btf_func_model { u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; +static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg) +{ + return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64); +} + /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. @@ -1684,12 +1743,21 @@ static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif +/* The contexts that a global function is called in, and is verified for. */ +enum bpf_func_context { + BPF_FUNC_CTX_NON_SLEEPABLE, + BPF_FUNC_CTX_SLEEPABLE, + /* Sleepable code has disabled preemption or interrupts: no RCU read lock. */ + BPF_FUNC_CTX_NO_RCU_LOCK, + BPF_FUNC_CTX_CNT, +}; + struct bpf_func_info_aux { u16 linkage; bool unreliable; - /* Indexed by in_sleepable. */ - bool called[2]; - bool verified[2]; + bool called[BPF_FUNC_CTX_CNT]; + bool verified[BPF_FUNC_CTX_CNT]; + bool called_with_pkt_ptrs; }; enum bpf_jit_poke_reason { @@ -1721,6 +1789,7 @@ struct bpf_ctx_arg_aux { struct btf *btf; u32 btf_id; u32 ref_id; + u32 mem_size; bool refcounted; }; @@ -1749,12 +1818,18 @@ enum { }; struct bpf_stream { - atomic_t capacity; + refcount_t refcnt; + atomic_t capacity; /* bytes reserved against the stream limit */ + atomic_t readable; /* published bytes available to readers */ struct llist_head log; /* list of in-flight stream elements in LIFO order */ struct mutex lock; /* lock protecting backlog_{head,tail} */ struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */ struct llist_node *backlog_tail; /* tail of the list above */ + wait_queue_head_t waitq; + struct irq_work notify_work; + bool notify_used; /* notify_work was queued at least once */ + bool dead; }; struct bpf_stream_stage { @@ -1773,6 +1848,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -1792,6 +1868,9 @@ struct bpf_prog_aux { u32 ctx_arg_info_size; u32 max_rdonly_access; u32 max_rdwr_access; + /* [min, max): the bytes of a rdwr buffer that may be written */ + u32 min_rdwr_write; + u32 max_rdwr_write; u32 subprog_start; struct btf *attach_btf; struct bpf_ctx_arg_aux *ctx_arg_info; @@ -1807,24 +1886,28 @@ struct bpf_prog_aux { bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool tramp_linked; /* true if it was ever linked to a trampoline */ + bool sleepable_async_cb; /* has a bpf_wq or bpf_task_work callback */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; bool exception_cb; bool exception_boundary; - bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; + bool has_prologue; bool might_sleep; + bool can_unwind; bool kprobe_write_ctx; + /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */ + bool uses_arena_insns; struct { s32 keyring_serial; u8 keyring_type; u8 verdict; } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ - struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ + struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ @@ -1856,6 +1939,7 @@ struct bpf_prog_aux { char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); u16 stack_arg_sp_adjust; + u16 freplace_link_cnt; /* counts freplace links extending this prog */ #ifdef CONFIG_SECURITY void *security; #endif @@ -1892,8 +1976,9 @@ struct bpf_prog_aux { union { struct work_struct work; struct rcu_head rcu; + struct rcu_work rwork; }; - struct bpf_stream stream[2]; + struct bpf_stream *stream[2]; struct mutex st_ops_assoc_mutex; struct bpf_map __rcu *st_ops_assoc; }; @@ -1940,6 +2025,15 @@ struct bpf_prog { }; }; +/* + * Does the program, or a callback of it, run under RCU Tasks Trace protection + * only, outside of any RCU read side critical section? + */ +static inline bool bpf_prog_runs_sleepable(const struct bpf_prog *prog) +{ + return prog->sleepable || prog->aux->sleepable_async_cb; +} + struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; @@ -2137,6 +2231,18 @@ struct btf_member; * unloaded while in use. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models + * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment. + * 0 means the struct_ops type does not support cgroup attachment. + * If cgroup_atype is non-zero, the @reg and @unreg must be NULL + * because the attachment/detachment will be handled by the bpf core. + * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for + * a tasks_rcu gp before freeing the struct_ops map + * and its progs. It is unnecessary if the @unreg + * has waited for the correct rcu gp or the @unreg + * has ensured all struct_ops prog has finished running. + * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for + * both tasks_trace_rcu and regular rcu grace period. + * It is usually needed if the struct_ops has sleepable prog. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; @@ -2155,6 +2261,9 @@ struct bpf_struct_ops { struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; + int cgroup_atype; + bool free_after_tasks_rcu_gp; + bool free_after_mult_rcu_gp; }; /* Every member of a struct_ops type has an instance even a member is not @@ -2289,6 +2398,12 @@ u32 bpf_struct_ops_id(const void *kdata); int bpf_struct_ops_for_each_prog(const void *kdata, int (*cb)(struct bpf_prog *prog, void *data), void *data); +void *bpf_struct_ops_map_kdata(struct bpf_map *map); +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map); +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map); +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map, + struct bpf_map *expected_old_map); +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2353,6 +2468,32 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { } +static inline u32 bpf_struct_ops_id(void *kdata) +{ + return 0; +} +static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + return NULL; +} +static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + return 0; +} +static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + return NULL; +} +static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +{ + return false; +} +static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + return -EOPNOTSUPP; +} #endif @@ -2528,7 +2669,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, * since other cpus are walking the array of pointers in parallel. */ struct bpf_prog_array_item { - struct bpf_prog *prog; + union { + struct bpf_prog *prog; + void *kdata; + }; union { struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; u64 bpf_cookie; @@ -2570,6 +2714,7 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *include_prog, u64 bpf_cookie, struct bpf_prog_array **new_array); +struct bpf_prog *bpf_prog_dummy(void); struct bpf_run_ctx {}; @@ -2588,6 +2733,7 @@ struct bpf_trace_run_ctx { struct bpf_tramp_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; + int retval; struct bpf_run_ctx *saved_run_ctx; }; @@ -2821,8 +2967,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **page_array); +struct page *bpf_alloc_page(int nid, bool sleepable); +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable); +void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, struct mem_cgroup **new_memcg); @@ -3008,6 +3156,8 @@ typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); +typedef int (*bpf_iter_check_prog_t)(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, struct seq_file *seq); typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, @@ -3017,7 +3167,8 @@ typedef const struct bpf_func_proto * const struct bpf_prog *prog); enum bpf_iter_feature { - BPF_ITER_RESCHED = BIT(0), + BPF_ITER_RESCHED = BIT(0), /* may cond_resched() between objects */ + BPF_ITER_NOSLEEP = BIT(1), /* ->show() can't run sleepable progs */ }; #define BPF_ITER_CTX_ARG_MAX 2 @@ -3025,6 +3176,8 @@ struct bpf_iter_reg { const char *target; bpf_iter_attach_target_t attach_target; bpf_iter_detach_target_t detach_target; + /* may the program run on what the link is attached to? */ + bpf_iter_check_prog_t check_prog; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; bpf_iter_get_func_proto_t get_func_proto; @@ -3061,6 +3214,8 @@ void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq); int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map); int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, @@ -3218,7 +3373,7 @@ const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr); + u16 desc_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; @@ -3561,7 +3716,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { return -ENOTSUPP; } @@ -3667,6 +3822,7 @@ int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); +void bpf_prog_dev_bound_moved(struct bpf_prog *prog); void bpf_dev_bound_netdev_unregister(struct net_device *dev); static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) @@ -3728,6 +3884,10 @@ static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, return -EOPNOTSUPP; } +static inline void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ +} + static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) { } @@ -3858,6 +4018,8 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3878,6 +4040,16 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } @@ -4138,9 +4310,10 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs); void bpf_put_buffers(void); -void bpf_prog_stream_init(struct bpf_prog *prog); +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) @@ -4203,7 +4376,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void); int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog); int bpf_insn_array_ready(struct bpf_map *map); void bpf_insn_array_release(struct bpf_map *map); -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len); +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len); void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL @@ -4228,6 +4401,9 @@ static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) } } +#define BPF_EXIST_FLAGS (BPF_EXIST | BPF_NOEXIST) +#define BPF_CPU_FLAGS (BPF_F_CPU | BPF_F_ALL_CPUS) + static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) { u32 cpu;
diff --git a/include/linux/bpf_compiler_types.h b/include/linux/bpf_compiler_types.h new file mode 100644 index 0000000..5c9e71a --- /dev/null +++ b/include/linux/bpf_compiler_types.h
@@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BPF_COMPILER_TYPES_H +#define __LINUX_BPF_COMPILER_TYPES_H + +#include <linux/compiler_types.h> + +/* + * Qualifiers for the BPF verifier rather than address spaces: __arena marks a + * pointer into a BPF arena, __nullable a pointer that may be NULL. They reach + * BPF programs as BTF type tags on the members of kernel structs shared with + * them; to kernel code and to sparse the pointers are ordinary. + */ +#ifdef __CHECKER__ +# define __arena +# define __nullable +#else +# define __arena BTF_TYPE_TAG(arena) +# define __nullable BTF_TYPE_TAG(nullable) +#endif + +#endif /* __LINUX_BPF_COMPILER_TYPES_H */
diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h deleted file mode 100644 index a41e71d..0000000 --- a/include/linux/bpf_crypto.h +++ /dev/null
@@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ -#ifndef _BPF_CRYPTO_H -#define _BPF_CRYPTO_H - -struct bpf_crypto_type { - void *(*alloc_tfm)(const char *algo); - void (*free_tfm)(void *tfm); - int (*has_algo)(const char *algo); - int (*setkey)(void *tfm, const u8 *key, unsigned int keylen); - int (*setauthsize)(void *tfm, unsigned int authsize); - int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - unsigned int (*ivsize)(void *tfm); - unsigned int (*statesize)(void *tfm); - u32 (*get_flags)(void *tfm); - struct module *owner; - char name[14]; -}; - -int bpf_crypto_register_type(const struct bpf_crypto_type *type); -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type); - -#endif /* _BPF_CRYPTO_H */
diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 9e4f5c4..6c643ac 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h
@@ -104,6 +104,19 @@ struct bpf_local_storage { refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ }; +/* + * Drop a reference got by refcount_inc_not_zero(&local_storage->owner_refcnt). + * bpf_local_storage_destroy() may have dropped the owner's reference meanwhile + * and wait for the count to reach zero, so this one can be the last. + */ +static inline void +bpf_local_storage_unpin_owner(struct bpf_local_storage *local_storage) +{ + bool last __maybe_unused; + + last = refcount_dec_and_test(&local_storage->owner_refcnt); +} + /* U16_MAX is much more than enough for sk local storage * considering a tcp_sock is ~2k. */
diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index dda272d..eb4a38e 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h
@@ -21,6 +21,13 @@ extern bool bpf_lsm_initialized __ro_after_init; #include <linux/lsm_hook_defs.h> #undef LSM_HOOK +/* + * Number of xattr slots the BPF LSM reserves in the array handed to + * security_inode_init_security(), i.e. the maximum number of labels + * a policy may attach to an inode while it is being created. + */ +#define BPF_LSM_INODE_INIT_XATTRS 2 + struct bpf_storage_blob { struct bpf_local_storage __rcu *storage; }; @@ -28,7 +35,7 @@ struct bpf_storage_blob { extern struct lsm_blob_sizes bpf_lsm_blob_sizes; int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog); + struct bpf_prog *prog); bool bpf_lsm_is_sleepable_hook(u32 btf_id); bool bpf_lsm_is_trusted(const struct bpf_prog *prog); @@ -71,7 +78,7 @@ static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog) } static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { return -EOPNOTSUPP; }
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b83ec99..5926f13 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h
@@ -9,6 +9,20 @@ #include <linux/filter.h> /* for MAX_BPF_STACK */ #include <linux/tnum.h> #include <linux/cnum.h> +#include <linux/math64.h> /* for div_s64_rem() */ + +/* + * Mathematical modulo, the residue of 'v' modulo 'step', normalized to [0, step). + * This differs from C's '%' operator, which truncates the quotient toward zero + * and so returns a remainder with the sign of the dividend (e.g. -1 % 3 == -1, not 2). + */ +static inline u16 imod(s64 v, u16 step) +{ + s32 rem; + + div_s64_rem(v, step, &rem); + return rem < 0 ? rem + step : rem; +} /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit @@ -19,11 +33,12 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -/* size of tmp_str_buf in bpf_verifier. - * we need at least 306 bytes to fit full stack mask representation - * (in the "-8,-16,...,-512" form) +/* + * size of tmp_str_buf in bpf_verifier. + * we need at least 1399 bytes to fit full stack mask representation + * (in the "-8,-16,...,-2048" form) */ -#define TMP_STR_BUF_LEN 320 +#define TMP_STR_BUF_LEN 1408 /* Patch buffer size */ #define INSN_BUF_SIZE 32 @@ -124,6 +139,13 @@ struct bpf_reg_state { */ struct cnum64 r64; /* 64-bit range as circular number */ struct cnum32 r32; /* 32-bit range as circular number */ + /* + * The value described by this register, interpreted as s64, lies on + * a line described by a linear equation base + step * k. + * Invariant: base < step. + */ + u16 base; + u16 step; /* For PTR_TO_PACKET, used to find other pointers with the same variable * offset, so they can share range knowledge. * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we @@ -162,9 +184,9 @@ struct bpf_reg_state { * other registers. Kept outside the metadata union for ID remapping * during state comparisons. */ - u32 map_uid; + u32 map_uid:31; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ - bool precise; + u32 precise:1; }; static inline s64 reg_smin(const struct bpf_reg_state *reg) @@ -243,55 +265,18 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +/* + * Largest number of BPF_REG_SIZE stack slots a single frame can have, sized + * for the largest stack budget any JIT supports. A frame may use any part of + * its program's budget; check_max_stack_depth() enforces the budget on the + * combined depth of frames sharing the kernel stack and on each frame using + * a private stack. + */ +#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK_JIT / BPF_REG_SIZE) + /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 #define STACK_SLOT_SZ 4 -#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */ - -typedef struct { - u64 v[2]; -} spis_t; - -#define SPIS_ZERO ((spis_t){}) -#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }}) - -static inline bool spis_is_zero(spis_t s) -{ - return s.v[0] == 0 && s.v[1] == 0; -} - -static inline bool spis_equal(spis_t a, spis_t b) -{ - return a.v[0] == b.v[0] && a.v[1] == b.v[1]; -} - -static inline spis_t spis_or(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }}; -} - -static inline spis_t spis_and(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }}; -} - -static inline spis_t spis_not(spis_t s) -{ - return (spis_t){{ ~s.v[0], ~s.v[1] }}; -} - -static inline bool spis_test_bit(spis_t s, u32 slot) -{ - return s.v[slot / 64] & BIT_ULL(slot % 64); -} - -static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) -{ - u32 w; - - for (w = lo; w <= hi && w < STACK_SLOTS; w++) - mask->v[w / 64] |= BIT_ULL(w % 64); -} #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ @@ -343,6 +328,25 @@ struct bpf_retval_range { bool return_32bit; }; +/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ +#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) + +struct bpf_loop_iters { + u32 min_header_count; /* min number of times header is executed */ + u32 max_header_count; /* max number of times header is executed */ + unsigned long precise_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + +struct loop_stack_entry { + struct bpf_verifier_state *entry_state; + u32 min_header_count; + u32 max_header_count; + u32 loop_id:31; + u32 terminates:1; +}; + +#define LOOP_STACK_SIZE 16 + /* state of the program: * type of all registers and stack info */ @@ -375,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. @@ -388,6 +397,11 @@ struct bpf_func_state { u32 callback_depth; /* Instructions processed in this frame and callees on the current path. */ u32 insns_subtotal; + /* + * Control-flow loop nesting at the current insn within this frame's + * subprogram (loops never cross subprogram boundaries). + */ + u32 loop_stack_cnt; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -405,6 +419,7 @@ struct bpf_func_state { u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */ struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */ + struct loop_stack_entry *loop_stack; }; #define MAX_CALL_FRAMES 16 @@ -421,30 +436,37 @@ enum { INSN_F_STACK_ARG_ACCESS = BIT(3), }; +/* Registers linked to one jump condition that a history entry can record */ +#define BPF_LINKED_REGS_MAX 5 + struct bpf_jmp_history_entry { /* insn idx can't be bigger than 1 million */ u32 idx : 20; u32 frame : 4; /* stack access frame number */ - u32 spi : 6; /* stack slot index (0..63) */ - u32 : 2; - u32 prev_idx : 20; /* special INSN_F_xxx flags */ u32 flags : 4; - u32 : 8; + u32 : 4; + u32 prev_idx : 20; + u32 spi : 12; /* stack slot index */ /* - * additional registers that need precision tracking when this - * jump is backtracked, vector of five 11-bit records + * Scalar registers and spilled scalars linked to the condition of + * this jump, which need precision tracking together when the jump is + * backtracked. Each is packed as 4 bits of frame number, one bit + * telling a register from a stack slot and 11 bits of register or + * slot index, see linked_regs_pack(). */ - u64 linked_regs; + u16 linked_regs[BPF_LINKED_REGS_MAX]; + u8 linked_regs_cnt; }; static_assert(MAX_CALL_FRAMES <= (1 << 4)); -static_assert(MAX_BPF_STACK / 8 <= (1 << 6)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); /* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \ - MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ + MAX_CALL_FRAMES) + struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -510,8 +532,13 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; + /* The callee left with unwind_exit. Next insn is unwind_goto. */ + bool unwound; - /* first and last insn idx of this verifier state */ + /* + * First and last insn idx of this verifier state. + * last_insn_idx is -1 if no instructions have been executed yet. + */ u32 first_insn_idx; u32 last_insn_idx; /* if this state is a backedge state then equal_state @@ -530,12 +557,31 @@ struct bpf_verifier_state { u32 may_goto_depth; }; +/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */ +static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame) +{ + return frame->allocated_stack / BPF_REG_SIZE; +} + +/* + * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8). + * The caller must ensure spi < bpf_stack_nr_slots(frame), see bpf_grow_stack_state(). + */ +static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi) +{ + return &frame->stack[spi]; +} + static inline struct bpf_reg_state * bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask) { - if (slot < frame->allocated_stack / BPF_REG_SIZE && - (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask) - return &frame->stack[slot].spilled_ptr; + struct bpf_stack_state *ss; + + if (slot >= bpf_stack_nr_slots(frame)) + return NULL; + ss = bpf_stack_slot(frame, slot); + if ((1 << ss->slot_type[BPF_REG_SIZE - 1]) & mask) + return &ss->spilled_ptr; return NULL; } @@ -551,7 +597,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ #define bpf_for_each_spilled_reg(iter, frame, reg, mask) \ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask); \ - iter < frame->allocated_stack / BPF_REG_SIZE; \ + iter < bpf_stack_nr_slots(frame); \ iter++, reg = bpf_get_spilled_reg(iter, frame, mask)) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */ @@ -576,7 +622,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \ if (!__reg) \ continue; \ - __stack = &__state->stack[___j]; \ + __stack = bpf_stack_slot(__state, ___j); \ (void)(__expr); \ } \ __stack = NULL; \ @@ -640,6 +686,52 @@ struct bpf_iarray { u32 items[]; }; +#define iarray_for_each(item, arr) \ + for (int ___idx = 0; \ + ___idx < (arr)->cnt && ({ item = (arr)->items[___idx]; 1; }); \ + ___idx++) + +#define MAX_BACKEDGES 16 +#define MAX_LOOP_EXITS 256 + +/* Number of exit conditions dominating a backedge. */ +#define MAX_LATCHES 4 + +struct bpf_backedge { + int from; + /* + * Conditional jumps within the loop that dominate 'from' and have + * a successor outside the loop, nearest first. latches[0] is the + * latch; the remaining entries are additional exit conditions. + * Each usable condition gives an upper bound on the iteration count, + * so keeping only a subset is sound. + */ + int latches[MAX_LATCHES]; + u8 latches_cnt; +}; + +struct bpf_loop_exit { + int from; /* instruction inside the loop */ + int to; /* instruction outside the loop */ +}; + +struct bpf_loop { + struct bpf_backedge backedges[MAX_BACKEDGES]; + /* edges exiting from this loop, includes edges from nested loops */ + struct bpf_loop_exit *exits; + int backedges_cnt; + int exits_cnt; + bool irreducible; + bool backedges_overflow; + bool exits_overflow; + /* + * Loop-entry registers used to compute stack addresses for loads, + * stores and calls requiring fixed stack offsets, in this and nested + * loops. + */ + unsigned long store_base_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + struct bpf_insn_aux_data { union { enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ @@ -671,45 +763,64 @@ struct bpf_insn_aux_data { /* remember the offset of node field within type to rewrite */ u64 insert_off; }; - struct bpf_iarray *jt; /* jump table for gotox or bpf_tailcall call instruction */ struct btf_struct_meta *kptr_struct_meta; u64 map_key_state; /* constant (32 bit) key tracking for maps */ + /* + * Per-instruction summary of stack slots in the current frame + * that this instruction may write to. + */ + DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); + DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS); + u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ - bool nospec; /* do not execute this instruction speculatively */ - bool nospec_result; /* result is unsafe under speculation, nospec must follow */ - bool zext_dst; /* this insn zero extends dst reg */ - bool needs_zext; /* alu op needs to clear upper bits */ - bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ - bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ - bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */ - bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ - u8 alu_state; /* used in combination with alu_limit */ + u64 nospec:1; /* do not execute this instruction speculatively */ + u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */ + u64 zext_dst:1; /* this insn zero extends dst reg */ + u64 needs_zext:1; /* alu op needs to clear upper bits */ + u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */ + u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ + u64 is_iter_next:1; /* bpf_iter_<type>_next() kfunc call */ + u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ + u64 arena_scalar:1; /* ldx/stx/st/atomic through a number, it's an address in arena */ + u64 bb_end:1; /* last instruction of a basic block */ + u64 need_scev:1; + u64 alu_state:5; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. */ - u8 fastcall_pattern:1; + u64 fastcall_pattern:1; /* for CALL instructions, a number of spill/fill pairs in the * bpf_fastcall pattern. */ - u8 fastcall_spills_num:3; - u8 arg_prog:4; + u64 fastcall_spills_num:3; + u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ - u32 jmp_point:1; - u32 prune_point:1; + /* below flags are initialized once */ + u64 jmp_point:1; + u64 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - u32 force_checkpoint:1; + u64 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - u32 calls_callback:1; - u32 indirect_target:1; /* if it is an indirect jump target */ + u64 calls_callback:1; + u64 indirect_target:1; /* if it is an indirect jump target */ + u64 non_stack_access:1; /* instruction can access non-stack memory */ /* true if some jump or call instruction targets this instruction */ - u32 jump_target:1; + u64 jump_target:1; + + unsigned int orig_idx; /* original instruction index, initialized once */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -729,12 +840,23 @@ struct bpf_insn_aux_data { u16 const_reg_map_mask; u16 const_reg_subprog_mask; u32 const_reg_vals[10]; + /* + * Header index of the innermost loop containing this instruction, -1 if none. + * For a loop header, identifies the parent loop instead. + */ + s32 loop_header; + /* additional information about the loop if this instruction is a loop header */ + struct bpf_loop *loop; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ #define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */ -#define BPF_VERIFIER_TMP_LOG_SIZE 1024 +/* + * Longest line the verifier log can carry: a full stack mask of + * MAX_BPF_STACK_SLOTS slots, see TMP_STR_BUF_LEN, plus its prefix. + */ +#define BPF_VERIFIER_TMP_LOG_SIZE 2048 struct bpf_verifier_log { /* Logical start and end positions of a "log window" of the verifier log. @@ -787,6 +909,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo #define BPF_MAX_SUBPROGS 256 +/* + * A pointer to a static subprog in the value of a frozen read-only array map: + * a 64-bit value that is the offset in bytes of the first instruction of + * the subprog in the program. + */ +struct bpf_func_ptr { + struct bpf_map *map; + u32 map_off; /* offset of the pointer in the value of the map */ + u32 orig_off; /* what the map has: the first instruction of the subprog */ + u32 xlated_off; /* the same after instructions were patched and removed */ +}; + +/* the subprog that a bpf_func_ptr pointed to was removed as dead code */ +#define BPF_FUNC_PTR_DELETED ((u32)-1) + struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { @@ -806,7 +943,12 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ - u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ + /* + * Index of one of the BPF_EXIT instructions in this subprogram, or + * U32_MAX when it has none. + */ + u32 exit_idx; + struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; u32 insns_total; @@ -817,17 +959,20 @@ struct bpf_subprog_info { s16 fastcall_stack_off; bool has_tail_call: 1; bool might_throw: 1; + bool can_unwind: 1; /* has unwind_exit */ bool tail_call_reachable: 1; bool has_ld_abs: 1; bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:4; + u8 arg_slot_cnt:4; enum priv_stack_mode priv_stack_mode; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; @@ -837,8 +982,8 @@ struct bpf_subprog_info { static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) { - if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) - return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; return 0; } @@ -849,7 +994,7 @@ struct backtrack_state { struct bpf_verifier_env *env; u32 frame; u32 reg_masks[MAX_CALL_FRAMES]; - u64 stack_masks[MAX_CALL_FRAMES]; + unsigned long stack_masks[MAX_CALL_FRAMES][BITS_TO_LONGS(MAX_BPF_STACK_SLOTS)]; u8 stack_arg_masks[MAX_CALL_FRAMES]; }; @@ -905,6 +1050,7 @@ struct bpf_scc_info { }; struct bpf_liveness; +struct scev; struct bpf_fd_array { union { @@ -928,6 +1074,7 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ + bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. @@ -954,6 +1101,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ + bool no_rcu_read_lock; bool seen_exception; bool signature; u32 insn_aux_data_len; @@ -961,9 +1110,24 @@ struct bpf_verifier_env { const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; struct bpf_diag *diag; + struct bpf_func_proto bpf_subprog_scratch; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Pointers to static subprogs found in frozen read-only maps of the + * program, see resolve_func_ptrs(). Sorted by map and map_off. + */ + struct bpf_func_ptr *func_ptrs; + u32 func_ptr_cnt; + bool has_callx; + /* + * Call graph edges created by callx instructions. A bitmap of + * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) + * is set when the main verification pass sees 'caller' calling 'callee' + * via callx. Allocated when the first such edge is recorded. + */ + unsigned long *callx_edges; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; @@ -976,14 +1140,17 @@ struct bpf_verifier_env { * see bpf_subprog_info->postorder_start. */ int *insn_postorder; + int *postorder_nums; int cur_stack; /* current position in the insn_postorder vector */ int cur_postorder; + u32 gotox_edges; + bool subprog_jts_ready; } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; - /* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */ - struct arg_track **callsite_at_stack; + /* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */ + struct spill_snapshot **callsite_at_stack; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ @@ -992,6 +1159,8 @@ struct bpf_verifier_env { u32 prev_jmps_processed, jmps_processed; /* maximum combined stack depth */ u32 max_stack_depth; + /* stack budget of the program, see bpf_prog_stack_limit() */ + u32 stack_limit; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -1027,7 +1196,7 @@ struct bpf_verifier_env { */ u32 scratched_regs; /* Same as scratched_regs but for stack slots */ - u64 scratched_stack_slots; + DECLARE_BITMAP(scratched_stack_slots, MAX_BPF_STACK_SLOTS); u64 prev_log_pos, prev_insn_print_pos; /* buffer used to temporary hold constants as scalar registers */ struct bpf_reg_state fake_reg[1]; @@ -1047,6 +1216,8 @@ struct bpf_verifier_env { u32 scc_cnt; struct bpf_iarray *succ; struct bpf_iarray *gotox_tmp_buf; + int *idoms; + struct scev *scev; }; static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) @@ -1059,8 +1230,13 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { - u8 num_params; + u8 arg_slot_cnt; bool is_void; bool fastcall; }; @@ -1083,6 +1259,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) insn->src_reg == BPF_PSEUDO_KFUNC_CALL; } +/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */ +static inline bool bpf_is_callx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL | BPF_X); +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, @@ -1097,6 +1279,7 @@ int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual); __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ @@ -1209,11 +1392,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info); +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx); +int bpf_split_cur_state(struct bpf_verifier_env *env); int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_jmp_history(struct bpf_verifier_state *state); @@ -1223,11 +1409,14 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs); + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt); void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm); void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); @@ -1241,6 +1430,24 @@ static inline int bpf_get_spi(s32 off) } /* + * Stack a program may use in total: combined over the frames of a call + * chain on the kernel stack, or per frame on a private stack. Any single + * frame may reach that deep. Only a JIT that lays out such frames may go + * beyond MAX_BPF_STACK, the interpreter's frame size, and only one whose + * tail calls let the target set up its own frame: without subprogram + * tail calls, do_misc_fixups() gives every program with tail calls a + * MAX_BPF_STACK frame, which a deeper frame would overrun. + */ +static inline u32 bpf_prog_stack_limit(const struct bpf_prog *prog) +{ + /* an offloaded program never runs on the host JIT, whatever it supports */ + if (prog->jit_requested && !bpf_prog_is_offloaded(prog->aux) && + bpf_jit_supports_large_stack() && bpf_jit_supports_subprog_tailcalls()) + return MAX_BPF_STACK_JIT; + return MAX_BPF_STACK; +} + +/* * Return the function state a stack pointer register refers to. frameno * shares storage with other pointer metadata, so return NULL for any * other register type instead of indexing frame[] with aliased bytes. @@ -1273,6 +1480,15 @@ static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack) return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; } +static inline int bpf_spill_size(const struct bpf_stack_state *stack) +{ + int i, spill_size = 0; + + for (i = BPF_REG_SIZE - 1; i >= 0 && stack->slot_type[i] == STACK_SPILL; i--) + spill_size++; + return spill_size; +} + static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack) { return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE; @@ -1290,12 +1506,7 @@ static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] |= 1ull << slot; -} - -static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) -{ - bt->stack_masks[frame] |= mask; + __set_bit(slot, bt->stack_masks[frame]); } static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) @@ -1310,10 +1521,17 @@ static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u3 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { - return bt->stack_masks[frame] & (1ull << slot); + return test_bit(slot, bt->stack_masks[frame]); } bool bpf_map_is_rdonly(const struct bpf_map *map); +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt); +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt); +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len); +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len); int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx); @@ -1441,7 +1659,7 @@ static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) { - env->scratched_stack_slots |= 1ULL << spi; + __set_bit(spi, env->scratched_stack_slots); } static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) @@ -1449,27 +1667,28 @@ static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) return (env->scratched_regs >> regno) & 1; } -static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u32 spi) { - return (env->scratched_stack_slots >> regno) & 1; + return test_bit(spi, env->scratched_stack_slots); } static inline bool verifier_state_scratched(const struct bpf_verifier_env *env) { - return env->scratched_regs || env->scratched_stack_slots; + return env->scratched_regs || + !bitmap_empty(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline void mark_verifier_state_clean(struct bpf_verifier_env *env) { env->scratched_regs = 0U; - env->scratched_stack_slots = 0ULL; + bitmap_zero(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } /* Used for printing the entire verifier state. */ static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) { env->scratched_regs = ~0U; - env->scratched_stack_slots = ~0ULL; + bitmap_fill(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size) @@ -1504,14 +1723,25 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); + int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); +void bpf_free_subprog_jts(struct bpf_verifier_env *env); int bpf_compute_postorder(struct bpf_verifier_env *env); int bpf_compute_scc(struct bpf_verifier_env *env); @@ -1540,12 +1770,15 @@ struct ref_obj_desc { }; /* - * A memory argument a call fills in. The verifier allows the stack to be uninitialized if - * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes + * variable-sized buffers from raw mode without losing the output annotation. + * size records constant ranges to mark initialized after checking all arguments, + * only when the caller is allowed to read uninitialized stack memory. */ struct arg_raw_mem_desc { - u8 regno; - int size; + u16 mask; + u16 var_size_mask; + int size[MAX_BPF_FUNC_ARGS]; }; /* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ @@ -1565,6 +1798,7 @@ struct bpf_call_arg_meta { struct btf *btf; u32 func_id; const struct bpf_func_proto *fn; + const struct btf_type *func_proto; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1572,20 +1806,25 @@ struct bpf_call_arg_meta { struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; struct ret_mem_desc ret_mem; + struct arg_raw_mem_desc arg_raw_mem; + + /* Only set by subprog */ + bool subprog_may_change_pkt; /* Only set by kfunc */ bool r0_rdonly; u32 kfunc_flags; - const struct btf_type *func_proto; const char *func_name; struct arg_constant_desc arg_constant; + /* The type of the dynptr that the kfunc writes through */ + enum bpf_dynptr_type dynptr_write_type; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later * verification logic * bpf_obj_drop/bpf_percpu_obj_drop * Record the local kptr type to be drop'd - * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) + * bpf_refcount_acquire (via ARG_PTR_TO_REFCOUNTED_KPTR arg type) * Record the local kptr type to be refcount_incr'd and use * arg_owning_ref to determine whether refcount_acquire should be * fallible @@ -1593,7 +1832,6 @@ struct bpf_call_arg_meta { struct btf *arg_btf; u32 arg_btf_id; bool arg_owning_ref; - bool arg_prog; struct { struct btf_field *field; @@ -1611,7 +1849,6 @@ struct bpf_call_arg_meta { s64 const_map_key; struct btf *ret_btf; struct btf_field *kptr_field; - struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, @@ -1638,17 +1875,37 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn); void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn); bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs); -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); +/* Stack effects used for state pruning; must_write implies may_write. */ +struct arg_access_info { + u32 size; /* Maximum extent; U32_MAX if unknown. */ + u8 may_read:1; /* Incoming contents or initialization may be needed. */ + u8 may_write:1; + u8 must_write:1; /* Prior verifier state is destroyed throughout size. */ +}; + +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b); +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step); +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); @@ -1680,6 +1937,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 @@ -1690,30 +1962,34 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) +static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; struct bpf_func_proto proto; u32 func_id; - s32 imm; u16 offset; unsigned long addr; }; struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. JITs use the descriptor index stored in the finalized + * call's off field. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ -void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm); @@ -1726,6 +2002,7 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct bpf_attach_target_info *tgt_info); /* Functions in fixups.c, called from bpf_check() */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env); int bpf_optimize_bpf_loop(struct bpf_verifier_env *env); void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env); @@ -1738,4 +2015,23 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); +int bpf_flip_opcode(u32 opcode); +u8 bpf_rev_opcode(u8 opcode); + +int bpf_compute_idoms(struct bpf_verifier_env *env); +int bpf_compute_loops(struct bpf_verifier_env *env); +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx); +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header); + +int bpf_init_scev(struct bpf_verifier_env *env); +void bpf_free_scev(struct bpf_verifier_env *env); +int bpf_compute_scev(struct bpf_verifier_env *env); + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters); +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters); +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count); + #endif /* _LINUX_BPF_VERIFIER_H */
diff --git a/include/linux/btf.h b/include/linux/btf.h index 7c62ea1..b66e20a 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h
@@ -5,6 +5,7 @@ #define _LINUX_BTF_H 1 #include <linux/types.h> +#include <linux/module.h> #include <linux/bpfptr.h> #include <linux/bsearch.h> #include <linux/btf_ids.h> @@ -118,6 +119,19 @@ struct btf_show; struct btf_id_set; struct bpf_prog; +#define BTF_LINK_SHA256_LEN 32 + +/* + * A .BTF.link or .BTF.inline.link section identifies the module which + * carries a vmlinux BTF section. The module name is NUL terminated and the + * field is padded to __MODULE_NAME_LEN. + */ +struct btf_link { + char module_name[__MODULE_NAME_LEN]; + u8 sha256[BTF_LINK_SHA256_LEN]; + u32 btf_size; +} __packed; + typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id); struct btf_kfunc_id_set { @@ -236,6 +250,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); @@ -261,6 +276,11 @@ const char *btf_type_str(const struct btf_type *t); i < btf_type_vlen(datasec_type); \ i++, member++) +#define for_each_loc(i, locsec_type, member) \ + for (i = 0, member = btf_type_loc_secinfo(locsec_type); \ + i < btf_type_vlen(locsec_type); \ + i++, member++) + static inline bool btf_type_is_ptr(const struct btf_type *t) { return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; @@ -329,6 +349,21 @@ static inline u64 btf_enum64_value(const struct btf_enum64 *e) return ((u64)e->val_hi32 << 32) | e->val_lo32; } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_type_loc_secinfo(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + static inline bool btf_is_composite(const struct btf_type *t) { u16 kind = btf_kind(t); @@ -559,7 +594,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter { @@ -573,7 +608,8 @@ struct btf_field_iter { #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids, + __u32 **map_strs); int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, enum btf_field_iter_kind iter_kind); __u32 *btf_field_iter_next(struct btf_field_iter *it); @@ -603,6 +639,29 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, const struct btf *btf2, u32 id2); int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx); +#if IS_ENABLED(CONFIG_SYSFS) +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name); +bool sysfs_btf_update(struct bin_attribute *attr, void *data); +void sysfs_btf_remove(struct bin_attribute *attr); +#else +static inline struct bin_attribute * +sysfs_btf_add(const char *name, void *data, size_t data_size, bool mmap, + const char *lazy_module_name) +{ + return NULL; +} + +static inline bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return false; +} + +static inline void sysfs_btf_remove(struct bin_attribute *attr) +{ +} +#endif + static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t) { if (!btf_type_is_ptr(t)) @@ -624,7 +683,7 @@ static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) } static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf, - __u32 **map_ids) + __u32 **map_ids, __u32 **map_strs) { return -EOPNOTSUPP; }
diff --git a/include/linux/cnum.h b/include/linux/cnum.h index 49b7d0c..4916031 100644 --- a/include/linux/cnum.h +++ b/include/linux/cnum.h
@@ -39,6 +39,7 @@ u32 cnum32_umin(struct cnum32 cnum); u32 cnum32_umax(struct cnum32 cnum); s32 cnum32_smin(struct cnum32 cnum); s32 cnum32_smax(struct cnum32 cnum); +struct cnum32 cnum32_union(struct cnum32 a, struct cnum32 b); struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b); void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src); void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max); @@ -65,6 +66,7 @@ u64 cnum64_umin(struct cnum64 cnum); u64 cnum64_umax(struct cnum64 cnum); s64 cnum64_smin(struct cnum64 cnum); s64 cnum64_smax(struct cnum64 cnum); +struct cnum64 cnum64_union(struct cnum64 a, struct cnum64 b); struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b); void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src); void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max); @@ -78,5 +80,6 @@ bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner); struct cnum32 cnum32_from_cnum64(struct cnum64 cnum); struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b); +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step); #endif /* _LINUX_CNUM_H */
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 8fae2b7..afaf2c1 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h
@@ -60,7 +60,7 @@ struct dma_map_ops { int (*dma_supported)(struct device *dev, u64 mask); u64 (*get_required_mask)(struct device *dev); size_t (*max_mapping_size)(struct device *dev); - size_t (*opt_mapping_size)(void); + size_t (*max_opt_mapping_size)(void); unsigned long (*get_merge_boundary)(struct device *dev); }; @@ -143,7 +143,7 @@ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, static inline void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { - __free_pages(page, get_order(size)); + free_pages_exact(page_address(page), size); } #endif /* CONFIG_DMA_CMA*/
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index a3e8806..efde0e5 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h
@@ -207,7 +207,7 @@ int dma_set_coherent_mask(struct device *dev, u64 mask); u64 dma_get_required_mask(struct device *dev); bool dma_addressing_limited(struct device *dev); size_t dma_max_mapping_size(struct device *dev); -size_t dma_opt_mapping_size(struct device *dev); +size_t dma_max_opt_mapping_size(struct device *dev); unsigned long dma_get_merge_boundary(struct device *dev); struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, enum dma_data_direction dir, gfp_t gfp, unsigned long attrs); @@ -326,7 +326,7 @@ static inline size_t dma_max_mapping_size(struct device *dev) { return 0; } -static inline size_t dma_opt_mapping_size(struct device *dev) +static inline size_t dma_max_opt_mapping_size(struct device *dev) { return 0; }
diff --git a/include/linux/filter.h b/include/linux/filter.h index 39decde7..e0218e6 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h
@@ -98,6 +98,11 @@ struct ctl_table_header; /* BPF program can access up to 512 bytes of stack space. */ #define MAX_BPF_STACK 512 +/* + * Stack budget of a program on a JIT that lays out frames of that size. + * The interpreter and JITs without such support keep MAX_BPF_STACK. + */ +#define MAX_BPF_STACK_JIT 2048 /* Helper macros for filter block array initializers. */ @@ -249,6 +254,29 @@ static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn) .off = 0, \ .imm = 1 }) +static inline bool bpf_is_unwind_goto(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_UNWIND_GOTO; +} + +static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_EXIT) && insn->src_reg == BPF_UNWIND_EXIT; +} + +/* + * The caller of the main prog doesn't tell unwind_exit from exit. An extension + * is different: its caller is the caller of the subprog that it replaced. + * An exception callback returns from the frame of the main prog. After freplace + * or a tail call it returns to a prog that wasn't verified with the callback. + */ +static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog) +{ + if (prog->aux->exception_cb) + return false; + return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT; +} + static inline bool insn_is_zext(const struct bpf_insn *insn) { return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; @@ -1105,6 +1133,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen) offsetof(struct bpf_prog, insns[proglen])); } +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); + +/* + * Whether a JIT has to set up the arena base for @prog. A program bound to + * an arena need not touch it, libbpf binds the arena of an object to every + * program in it. The flag caches bpf_prog_has_arena_insns() for the function + * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). + */ +static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) +{ + return prog->aux->uses_arena_insns; +} + static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) { /* When classic BPF programs have been loaded and the arch @@ -1237,16 +1278,53 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_callx(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); +bool bpf_jit_supports_unwind(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); +bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); +bool bpf_jit_supports_large_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); + +struct bpf_jit_arg_abi { + /* Argument registers of the kernel convention. */ + u8 nr_arg_regs; + /* Round the register number up to an even one for 16-byte alignment. */ + bool even_reg_align; + /* Round the stack slot up to an even one for 16-byte alignment. */ + bool even_stack_align; + /* An argument may straddle the last register and the stack. */ + bool split_at_boundary; + /* A later argument may reuse a register a stack-passed one skipped. */ + bool backfill_after_stack; +}; + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot); + +/* The JIT's scratch register, in place of an argument slot. */ +#define BPF_JIT_ARG_TMP 0xff + +/* Every argument slot moves at most once, and the scratch goes out and back. */ +#define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) + +struct bpf_jit_arg_move { + u8 dst; + u8 src; +}; + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves); u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); @@ -1897,6 +1975,11 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde return XDP_REDIRECT; } +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags); +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags); + #ifdef CONFIG_NET int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len); int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index bd76a16..b041d91 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h
@@ -416,6 +416,7 @@ void free_ftrace_hash(struct ftrace_hash *hash); struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct); void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry); +struct ftrace_func_entry *ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); void ftrace_hash_remove(struct ftrace_hash *hash); /* The hash used to know what functions callbacks trace */
diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index 07ef1c8..1b39b27 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h
@@ -213,4 +213,10 @@ enum { PT_FEAT_X86_64_AMD_ENCRYPT_TABLES = PT_FEAT_FMT_START, }; +struct pt_bcm2712 { + struct pt_common common; + u8 bigpage_lg2; + u8 superpage_lg2; +}; + #endif
diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index dd0edd0..13ebb72 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h
@@ -346,6 +346,18 @@ struct pt_iommu_x86_64_hw_info { IOMMU_FORMAT(x86_64, x86_64_pt); +struct pt_iommu_bcm2712_cfg { + struct pt_iommu_cfg common; + u8 bigpage_lg2; + u8 superpage_lg2; +}; + +struct pt_iommu_bcm2712_hw_info { + phys_addr_t pt_base; +}; + +IOMMU_FORMAT(bcm2712, bcm2712pt); + #undef IOMMU_PROTOTYPES #undef IOMMU_FORMAT #endif
diff --git a/include/linux/gfp.h b/include/linux/gfp.h index b62e2fd..63a7654 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h
@@ -325,7 +325,7 @@ void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1); void free_pages_exact(void *virt, size_t size); -__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); +void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); #define alloc_pages_exact_nid(...) \ alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__))
diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 9e109d9..e61f9a4 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h
@@ -1299,6 +1299,14 @@ struct hv_device { #define device_to_hv_device(d) container_of_const(d, struct hv_device, device) #define drv_to_hv_drv(d) container_of_const(d, struct hv_driver, driver) +static inline u32 hv_build_logical_dev_id_prefix(struct hv_device *hdev) +{ + return ((u32)hdev->dev_instance.b[5] << 24) | + ((u32)hdev->dev_instance.b[4] << 16) | + ((u32)hdev->dev_instance.b[7] << 8) | + (hdev->dev_instance.b[6] & 0xf8u); +} + static inline void hv_set_drvdata(struct hv_device *dev, void *data) { dev_set_drvdata(&dev->device, data);
diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h index 060f6e2..fae5d50 100644 --- a/include/linux/iommu-dma.h +++ b/include/linux/iommu-dma.h
@@ -39,7 +39,7 @@ int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs); unsigned long iommu_dma_get_merge_boundary(struct device *dev); -size_t iommu_dma_opt_mapping_size(void); +size_t iommu_dma_max_opt_mapping_size(void); size_t iommu_dma_max_mapping_size(struct device *dev); void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, unsigned long attrs);
diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 05f7cee..ef50437 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h
@@ -900,7 +900,6 @@ int iommu_device_sysfs_add(struct iommu_device *iommu, void iommu_device_sysfs_remove(struct iommu_device *iommu); int iommu_device_link(struct iommu_device *iommu, struct device *link); void iommu_device_unlink(struct iommu_device *iommu, struct device *link); -int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); static inline struct iommu_device *dev_to_iommu_device(struct device *dev) { @@ -945,7 +944,6 @@ extern void iommu_detach_device(struct iommu_domain *domain, struct device *dev); extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev); -extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); extern int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp); int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 2c2fb39..299e75b 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h
@@ -63,20 +63,34 @@ #define GICV5_OUTER_SHARE 0b10 #define GICV5_INNER_SHARE 0b11 +#define GICV5_AIDR_COMPONENT_IRS 0b00 +#define GICV5_AIDR_COMPONENT_ITS 0b01 +#define GICV5_AIDR_COMPONENT_IWB 0b10 + +#define GICV5_AIDR_ARCH_MAJ_REV_V5 0 +#define GICV5_AIDR_ARCH_MIN_REV_V0 0 + /* * IRS registers and tables structures */ #define GICV5_IRS_IDR0 0x0000 #define GICV5_IRS_IDR1 0x0004 #define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 #define GICV5_IRS_IDR5 0x0014 #define GICV5_IRS_IDR6 0x0018 #define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 #define GICV5_IRS_CR0 0x0080 #define GICV5_IRS_CR1 0x0084 #define GICV5_IRS_SYNCR 0x00c0 #define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 #define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 #define GICV5_IRS_SPI_CFGR 0x0114 #define GICV5_IRS_SPI_STATUSR 0x0118 #define GICV5_IRS_PE_SELR 0x0140 @@ -86,11 +100,51 @@ #define GICV5_IRS_IST_CFGR 0x0190 #define GICV5_IRS_IST_STATUSR 0x0194 #define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 +#define GICV5_IRS_IDR0_IRSID GENMASK(31, 16) +#define GICV5_IRS_IDR0_SWE BIT(12) +#define GICV5_IRS_IDR0_MPAM BIT(11) +#define GICV5_IRS_IDR0_MEC BIT(10) +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_VIRT_ONE_N BIT(8) +#define GICV5_IRS_IDR0_ONE_N BIT(7) #define GICV5_IRS_IDR0_VIRT BIT(6) +#define GICV5_IRS_IDR0_PA_RANGE GENMASK(5, 2) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) + +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 +#define GICV5_IRS_IDR0_INT_DOM_NON_SECURE 0b01 +#define GICV5_IRS_IDR0_INT_DOM_EL3 0b10 +#define GICV5_IRS_IDR0_INT_DOM_REALM 0b11 #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) #define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16) +#define GICV5_IRS_IDR1_PE_CNT GENMASK(15, 0) #define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000 #define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001 @@ -106,13 +160,30 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(0), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(1), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(2), (r)) + +#define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) +#define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) +#define GICV5_IRS_IDR3_VMD_SZ GENMASK(4, 1) +#define GICV5_IRS_IDR3_VMD BIT(0) + +#define GICV5_IRS_IDR4_VPE_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR4_VPED_SZ GENMASK(5, 0) + #define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IIDR_PRODUCT_ID GENMASK(31, 20) +#define GICV5_IRS_IIDR_VARIANT GENMASK(19, 16) +#define GICV5_IRS_IIDR_REVISION GENMASK(15, 12) +#define GICV5_IRS_IIDR_IMPLEMENTER GENMASK(11, 0) + +#define GICV5_IRS_AIDR_COMPONENT GENMASK(11, 8) +#define GICV5_IRS_AIDR_ARCHMAJORREV GENMASK(7, 4) +#define GICV5_IRS_AIDR_ARCHMINORREV GENMASK(3, 0) #define GICV5_IRS_CR0_IDLE BIT(1) #define GICV5_IRS_CR0_IRSEN BIT(0) @@ -135,21 +206,39 @@ #define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) -#define GICV5_IRS_SPI_STATUSR_V BIT(1) -#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_VMR_VIRT BIT_ULL(63) +#define GICV5_IRS_SPI_VMR_VM_ID GENMASK_ULL(15, 0) #define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_DOMAINR_DOMAIN GENMASK(1, 0) + +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_SECURE 0b00 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE 0b01 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_EL3 0b10 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_REALM 0b11 + +#define GICV5_IRS_SPI_RESAMPLER_ID GENMASK(23, 0) + #define GICV5_IRS_SPI_CFGR_TM BIT(0) +#define GICV5_IRS_SPI_CFGR_TM_EDGE 0b0 +#define GICV5_IRS_SPI_CFGR_TM_LEVEL 0b1 + +#define GICV5_IRS_SPI_STATUSR_V BIT(1) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) + #define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0) +#define GICV5_IRS_PE_STATUSR_ONLINE BIT(2) #define GICV5_IRS_PE_STATUSR_V BIT(1) #define GICV5_IRS_PE_STATUSR_IDLE BIT(0) #define GICV5_IRS_PE_CR0_DPS BIT(0) -#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL #define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) #define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) @@ -167,15 +256,111 @@ #define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01 #define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10 -#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) -#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) #define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0) +#define GICV5_IRS_VMT_BASER_ADDR GENMASK_ULL(55, 3) +#define GICV5_IRS_VMT_BASER_ADDR_SHIFT 3ULL +#define GICV5_IRS_VMT_BASER_VALID BIT_ULL(0) + +#define GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL 0b1 +#define GICV5_IRS_VMT_CFGR_STRUCTURE_LINEAR 0b0 + +#define GICV5_IRS_VMT_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_VMT_CFGR_VM_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_VMT_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VPE_SELR_S BIT_ULL(63) +#define GICV5_IRS_VPE_SELR_VPE_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VPE_SELR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VPE_DBR_DBV BIT_ULL(63) +#define GICV5_IRS_VPE_DBR_REQ_DB BIT_ULL(62) +#define GICV5_IRS_VPE_DBR_DBPM GENMASK_ULL(36, 32) +#define GICV5_IRS_VPE_DBR_INTID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_HPPIR_HPPIV BIT_ULL(32) +#define GICV5_IRS_VPE_HPPIR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VPE_HPPIR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_CR0_DPS BIT(0) + +#define GICV5_IRS_VPE_STATUSR_V BIT(1) +#define GICV5_IRS_VPE_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VM_DBR_EN BIT_ULL(63) +#define GICV5_IRS_VM_DBR_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VM_SELR_VM_ID GENMASK(15, 0) + +#define GICV5_IRS_VM_STATUSR_V BIT(1) +#define GICV5_IRS_VM_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VMAP_L2_VMTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VMTR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VMR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VMR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VMR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VISTR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VISTR_TYPE GENMASK_ULL(31, 29) + +#define GICV5_IRS_VMAP_L2_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_L2_VISTR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VMAP_L2_VISTR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VMAP_VPER_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VPER_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VPER_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SAVE_VMR_VM_ID GENMASK_ULL(15, 0) +#define GICV5_IRS_SAVE_VMR_Q BIT_ULL(62) +#define GICV5_IRS_SAVE_VMR_S BIT_ULL(63) + +#define GICV5_IRS_SAVE_VM_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SAVE_VM_STATUSR_Q BIT(1) + +#define GICV5_IRS_MEC_IDR_MECIDSIZE GENMASK(3, 0) + +#define GICV5_IRS_MEC_MECID_R_MECID GENMASK(15, 0) + +#define GICV5_IRS_MPAM_IDR_HAS_MPAM_SP BIT(24) +#define GICV5_IRS_MPAM_IDR_PMG_MAX GENMASK(23, 16) +#define GICV5_IRS_MPAM_IDR_PARTID_MAX GENMASK(15, 0) + +#define GICV5_IRS_MPAM_PARTID_R_IDLE BIT(31) +#define GICV5_IRS_MPAM_PARTID_R_MPAM_SP GENMASK(25, 24) +#define GICV5_IRS_MPAM_PARTID_R_PMG GENMASK(23, 16) +#define GICV5_IRS_MPAM_PARTID_R_PARTID GENMASK(15, 0) + +#define GICV5_IRS_SWERR_STATUSR_IMP_EC GENMASK_ULL(31, 24) +#define GICV5_IRS_SWERR_STATUSR_EC GENMASK_ULL(23, 16) +#define GICV5_IRS_SWERR_STATUSR_OF BIT_ULL(3) +#define GICV5_IRS_SWERR_STATUSR_S1V BIT_ULL(2) +#define GICV5_IRS_SWERR_STATUSR_S0V BIT_ULL(1) +#define GICV5_IRS_SWERR_STATUSR_V BIT_ULL(0) + +#define GICV5_IRS_SWERR_SYNDROMER0_VIRTUAL BIT_ULL(63) +#define GICV5_IRS_SWERR_SYNDROMER0_TYPE GENMASK_ULL(62, 60) +#define GICV5_IRS_SWERR_SYNDROMER0_ID GENMASK_ULL(55, 32) +#define GICV5_IRS_SWERR_SYNDROMER0_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SWERR_SYNDROMER1_ADDR GENMASK_ULL(55, 3) + #define GICV5_ISTL1E_VALID BIT_ULL(0) +#define GICV5_IRS_ISTL1E_SIZE 8UL #define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12) +#define GICV5_IRS_SETLPIR 0x0000 +#define GICV5_IRS_SETLPIR_ID GENMASK(23, 0) + /* * ITS registers and tables structures */ @@ -298,6 +483,42 @@ #define GICV5_GSI_IWB_WIRE GENMASK(15, 0) /* + * CoreSight identification registers - ordered by increasing offset. + */ +#define GICV5_CORESIGHT_DEVARCH 0xffbc +#define GICV5_CORESIGHT_PIDR4 0xffd0 +#define GICV5_CORESIGHT_PIDR5 0xffd4 +#define GICV5_CORESIGHT_PIDR6 0xffd8 +#define GICV5_CORESIGHT_PIDR7 0xffdc +#define GICV5_CORESIGHT_PIDR0 0xffe0 +#define GICV5_CORESIGHT_PIDR1 0xffe4 +#define GICV5_CORESIGHT_PIDR2 0xffe8 +#define GICV5_CORESIGHT_PIDR3 0xffec +#define GICV5_CORESIGHT_CIDR0 0xfff0 +#define GICV5_CORESIGHT_CIDR1 0xfff4 +#define GICV5_CORESIGHT_CIDR2 0xfff8 +#define GICV5_CORESIGHT_CIDR3 0xfffc + +#define GICV5_CORESIGHT_DEVARCH_VAL \ + (FIELD_PREP(GENMASK(31, 21), 0x23b) | \ + BIT(20) | \ + 0x5a19) + +#define GICV5_CORESIGHT_PIDR4_JEP106_CONT 0x04 +#define GICV5_CORESIGHT_PIDR5_RES0 0x00 +#define GICV5_CORESIGHT_PIDR6_RES0 0x00 +#define GICV5_CORESIGHT_PIDR7_RES0 0x00 +#define GICV5_CORESIGHT_PIDR0_PART_0 0x4b +#define GICV5_CORESIGHT_PIDR1_DES_0_PART_1 0xb0 +#define GICV5_CORESIGHT_PIDR2_DES_1 0x0b +#define GICV5_CORESIGHT_PIDR3_REVAND_CMOD 0x00 + +#define GICV5_CORESIGHT_CIDR0_VAL 0x0d +#define GICV5_CORESIGHT_CIDR1_VAL 0xf0 +#define GICV5_CORESIGHT_CIDR2_VAL 0x05 +#define GICV5_CORESIGHT_CIDR3_VAL 0xb1 + +/* * Global Data structures and functions */ struct gicv5_chip_data { @@ -332,6 +553,8 @@ struct gicv5_irs_chip_data { raw_spinlock_t spi_config_lock; }; +#define IRS_FLAGS_NON_COHERENT BIT(0) + static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset, const char *reg_s, u32 mask, u32 *val) @@ -379,6 +602,7 @@ void __init gicv5_free_lpi_domain(void); int gicv5_irs_of_probe(struct device_node *parent); int gicv5_irs_acpi_probe(void); +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void); void gicv5_irs_remove(void); int gicv5_irs_enable(void); void gicv5_irs_its_probe(void); @@ -387,10 +611,13 @@ int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid); struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id); int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); +unsigned int gicv5_irs_l2_sz(u32 l2sz); void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ struct gicv5_vpe { + int db; + bool db_fired; bool resident; }; @@ -429,4 +656,15 @@ void gicv5_deinit_lpis(void); void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); + +enum gicv5_vcpu_cmd { + VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ + VMTE_MAKE_VALID, /* Make the VMTE valid */ + VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + VPE_MAKE_VALID, /* No corresponding invalid */ + SPI_VIST_MAKE_VALID, /* No corresponding invalid */ + LPI_VIST_MAKE_VALID, /* Triggered by a guest */ + LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ +}; + #endif
diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h index 67d9d96..f05370e 100644 --- a/include/linux/irqchip/arm-vgic-info.h +++ b/include/linux/irqchip/arm-vgic-info.h
@@ -38,6 +38,11 @@ struct gic_kvm_info { bool has_v4_1; /* Deactivation impared, subpar stuff */ bool no_hw_deactivation; + /* GICv5 IRS base */ + struct { + void __iomem *base; + bool non_coherent; + } gicv5_irs; }; #ifdef CONFIG_KVM
diff --git a/include/linux/key.h b/include/linux/key.h index e40b526..dae5ba2 100644 --- a/include/linux/key.h +++ b/include/linux/key.h
@@ -445,6 +445,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c844def..e7fc94e 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h
@@ -45,6 +45,10 @@ #include <asm/kvm_host.h> #include <linux/kvm_dirty_ring.h> +#define __KVM_DEV_STRING(name, x) __stringify(name ## _ ## x) +#define _KVM_DEV_STRING(name, x) __KVM_DEV_STRING(name, x) +#define KVM_DEV_STRING(x) _KVM_DEV_STRING(KVM_DEV_NAME, x) + #ifndef KVM_MAX_VCPU_IDS #define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS #endif @@ -562,7 +566,7 @@ static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) /* * The memory barrier ensures a previous write to vcpu->requests cannot * be reordered with the read of vcpu->mode. It pairs with the general - * memory barrier following the write of vcpu->mode in VCPU RUN. + * memory barrier following the write of vcpu->mode in KVM_RUN. */ smp_mb__before_atomic(); return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); @@ -722,31 +726,14 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline bool kvm_arch_has_private_mem(struct kvm *kvm) -{ - return false; -} -#endif - -#ifdef CONFIG_KVM_GUEST_MEMFD -bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); - -static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) -{ - u64 flags = GUEST_MEMFD_FLAG_MMAP; - - if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) - flags |= GUEST_MEMFD_FLAG_INIT_SHARED; - - return flags; -} -#endif - #ifndef kvm_arch_has_readonly_mem static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) { - return IS_ENABLED(CONFIG_HAVE_KVM_READONLY_MEM); +#ifdef __KVM_HAVE_READONLY_MEM + return true; +#else + return false; +#endif } #endif @@ -784,6 +771,12 @@ struct kvm { * kvm_swap_active_memslots(). */ struct mutex slots_arch_lock; + /* + * Back-reference to the VM file for subsystems (e.g., VFIO). Holds no + * reference to avoid pinning the VM. Callers storing the file must + * take their own reference. + */ + struct file *file; struct mm_struct *mm; /* userspace tied to this vm */ unsigned long nr_memslot_pages; /* The two memslot sets - active and inactive (per address space) */ @@ -832,7 +825,7 @@ struct kvm { struct kvm_vm_stat stat; struct kvm_arch arch; refcount_t users_count; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; spinlock_t ring_lock; struct list_head coalesced_zones; @@ -854,6 +847,8 @@ struct kvm { gfn_t mmu_invalidate_range_start; gfn_t mmu_invalidate_range_end; + unsigned long gpc_invalidate_seq; + struct list_head devices; u64 manual_dirty_log_protect; struct dentry *debugfs_dentry; @@ -871,7 +866,7 @@ struct kvm { #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER struct notifier_block pm_notifier; #endif -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* Protected by slots_lock (for writes) and RCU (for reads) */ struct xarray mem_attr_array; #endif @@ -1080,7 +1075,18 @@ void kvm_exit(void); void kvm_get_kvm(struct kvm *kvm); bool kvm_get_kvm_safe(struct kvm *kvm); void kvm_put_kvm(struct kvm *kvm); -bool file_is_kvm(struct file *file); + +/* + * Architectures define kvm_file_to_kvm_arch to <arch> + * to get a typed, arch-namespaced helper: + * + * struct kvm *file_to_kvm_<arch>(struct file *file) + */ +#ifdef kvm_file_to_kvm_arch +#define kvm_file_to_kvm_fn CONCATENATE(file_to_kvm_, kvm_file_to_kvm_arch) +struct kvm *kvm_file_to_kvm_fn(struct file *file); +#endif /* kvm_file_to_kvm_arch */ + void kvm_put_kvm_no_destroy(struct kvm *kvm); static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) @@ -2464,7 +2470,7 @@ void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, struct kvm_kernel_irq_routing_entry *new); #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ -#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS +#ifdef __KVM_HAVE_INVALID_WAKEUPS /* If we wakeup during the poll time, was it a sucessful poll? */ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { @@ -2476,9 +2482,9 @@ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { return true; } -#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ +#endif /* __KVM_HAVE_INVALID_WAKEUPS */ -#ifdef CONFIG_HAVE_KVM_NO_POLL +#ifdef __KVM_HAVE_NO_POLL /* Callback that tells if we must not poll */ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); #else @@ -2486,18 +2492,18 @@ static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) { return false; } -#endif /* CONFIG_HAVE_KVM_NO_POLL */ +#endif /* __KVM_HAVE_NO_POLL */ void kvm_arch_guest_memory_reclaimed(struct kvm *kvm); -#ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE +#ifdef __KVM_HAVE_VCPU_RUN_PID_CHANGE int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); #else static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) { return 0; } -#endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ +#endif /* __KVM_HAVE_VCPU_RUN_PID_CHANGE */ #ifdef CONFIG_VIRT_XFER_TO_GUEST_WORK static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) @@ -2564,48 +2570,79 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) return slot->flags & KVM_MEMSLOT_GMEM_ONLY; } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn) { return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); } -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs); -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs); +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +extern bool gmem_in_place_conversion; +typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); +DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { - return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; + return static_call(__kvm_is_private_gfn)(kvm, gfn); } + +#elif defined kvm_arch_has_private_mem +#define gmem_in_place_conversion true +#define kvm_is_private_gfn kvm_gmem_is_private_gfn + #else -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +#define gmem_in_place_conversion false + +static inline bool kvm_arch_has_private_mem(struct kvm *kvm) { return false; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return false; +} +#endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn); +bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); + +static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) +{ + u64 flags = GUEST_MEMFD_FLAG_MMAP; + + if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) + flags |= GUEST_MEMFD_FLAG_INIT_SHARED; + + return flags; +} + int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order); + gfn_t gfn, kvm_pfn_t *pfn, int *max_order); #else static inline int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, - kvm_pfn_t *pfn, struct page **page, - int *max_order) + kvm_pfn_t *pfn, int *max_order) { KVM_BUG_ON(1, kvm); return -EIO; } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() false #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE @@ -2647,6 +2684,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu); long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif
diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index ba9e398..5f74f79 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h
@@ -33,6 +33,7 @@ * @kobj: kobject for sysfs resources * @node: list node for klp_object func_list * @stack_node: list node for klp_ops func_stack list + * @obj: back pointer to the owning object * @old_size: size of the old function * @new_size: size of the new function * @nop: temporary patch to use the original code again; dyn. allocated @@ -72,6 +73,7 @@ struct klp_func { struct kobject kobj; struct list_head node; struct list_head stack_node; + struct klp_object *obj; unsigned long old_size, new_size; bool nop; bool patched; @@ -86,6 +88,7 @@ struct klp_func { * @kobj: kobject for sysfs resources * @func_list: dynamic list of the function entries * @node: list node for klp_patch obj_list + * @patch: back pointer to the owning patch * @mod: kernel module associated with the patched object * (NULL for vmlinux) * @dynamic: temporary object for nop functions; dynamically allocated @@ -101,6 +104,7 @@ struct klp_object { struct kobject kobj; struct list_head func_list; struct list_head node; + struct klp_patch *patch; struct module *mod; bool dynamic; bool patched;
diff --git a/include/linux/mlx5/cq.h b/include/linux/mlx5/cq.h index 9d47cdc..a1c1447 100644 --- a/include/linux/mlx5/cq.h +++ b/include/linux/mlx5/cq.h
@@ -81,16 +81,6 @@ enum { enum { MLX5_CQE_OWNER_MASK = 1, - MLX5_CQE_REQ = 0, - MLX5_CQE_RESP_WR_IMM = 1, - MLX5_CQE_RESP_SEND = 2, - MLX5_CQE_RESP_SEND_IMM = 3, - MLX5_CQE_RESP_SEND_INV = 4, - MLX5_CQE_RESIZE_CQ = 5, - MLX5_CQE_SIG_ERR = 12, - MLX5_CQE_REQ_ERR = 13, - MLX5_CQE_RESP_ERR = 14, - MLX5_CQE_INVALID = 15, }; enum {
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 8cb321a..37727b5 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h
@@ -36,6 +36,7 @@ #include <linux/types.h> #include <rdma/ib_verbs.h> #include <linux/mlx5/mlx5_ifc.h> +#include <linux/mlx5/mlx5_ifc_macros.h> #include <linux/bitfield.h> #if defined(__LITTLE_ENDIAN) @@ -46,122 +47,6 @@ #error Host endianness not defined #endif -/* helper macros */ -#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0) -#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld) -#define __mlx5_bit_off(typ, fld) (offsetof(struct mlx5_ifc_##typ##_bits, fld)) -#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16) -#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32) -#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64) -#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0xf)) -#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f)) -#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) -#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld)) -#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) -#define __mlx5_16_mask(typ, fld) (__mlx5_mask16(typ, fld) << __mlx5_16_bit_off(typ, fld)) -#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits) - -#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8) -#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8) -#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32) -#define MLX5_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64) -#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8) -#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32) -#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8) -#define MLX5_ADDR_OF(typ, p, fld) ((void *)((u8 *)(p) + MLX5_BYTE_OFF(typ, fld))) - -/* insert a value to a struct */ -#define MLX5_SET(typ, p, fld, v) do { \ - u32 _v = v; \ - BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ - *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ - cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ - (~__mlx5_dw_mask(typ, fld))) | (((_v) & __mlx5_mask(typ, fld)) \ - << __mlx5_dw_bit_off(typ, fld))); \ -} while (0) - -#define MLX5_ARRAY_SET(typ, p, fld, idx, v) do { \ - BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 32); \ - MLX5_SET(typ, p, fld[idx], v); \ -} while (0) - -#define MLX5_SET_TO_ONES(typ, p, fld) do { \ - BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ - *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ - cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ - (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \ - << __mlx5_dw_bit_off(typ, fld))); \ -} while (0) - -#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\ -__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \ -__mlx5_mask(typ, fld)) - -#define MLX5_GET_PR(typ, p, fld) ({ \ - u32 ___t = MLX5_GET(typ, p, fld); \ - pr_debug(#fld " = 0x%x\n", ___t); \ - ___t; \ -}) - -#define __MLX5_SET64(typ, p, fld, v) do { \ - BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \ - *((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \ -} while (0) - -#define MLX5_SET64(typ, p, fld, v) do { \ - BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ - __MLX5_SET64(typ, p, fld, v); \ -} while (0) - -#define MLX5_ARRAY_SET64(typ, p, fld, idx, v) do { \ - BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ - __MLX5_SET64(typ, p, fld[idx], v); \ -} while (0) - -#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld))) - -#define MLX5_GET64_PR(typ, p, fld) ({ \ - u64 ___t = MLX5_GET64(typ, p, fld); \ - pr_debug(#fld " = 0x%llx\n", ___t); \ - ___t; \ -}) - -#define MLX5_GET16(typ, p, fld) ((be16_to_cpu(*((__be16 *)(p) +\ -__mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \ -__mlx5_mask16(typ, fld)) - -#define MLX5_SET16(typ, p, fld, v) do { \ - u16 _v = v; \ - BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 16); \ - *((__be16 *)(p) + __mlx5_16_off(typ, fld)) = \ - cpu_to_be16((be16_to_cpu(*((__be16 *)(p) + __mlx5_16_off(typ, fld))) & \ - (~__mlx5_16_mask(typ, fld))) | (((_v) & __mlx5_mask16(typ, fld)) \ - << __mlx5_16_bit_off(typ, fld))); \ -} while (0) - -/* Big endian getters */ -#define MLX5_GET64_BE(typ, p, fld) (*((__be64 *)(p) +\ - __mlx5_64_off(typ, fld))) - -#define MLX5_GET_BE(type_t, typ, p, fld) ({ \ - type_t tmp; \ - switch (sizeof(tmp)) { \ - case sizeof(u8): \ - tmp = (__force type_t)MLX5_GET(typ, p, fld); \ - break; \ - case sizeof(u16): \ - tmp = (__force type_t)cpu_to_be16(MLX5_GET(typ, p, fld)); \ - break; \ - case sizeof(u32): \ - tmp = (__force type_t)cpu_to_be32(MLX5_GET(typ, p, fld)); \ - break; \ - case sizeof(u64): \ - tmp = (__force type_t)MLX5_GET64_BE(typ, p, fld); \ - break; \ - } \ - tmp; \ - }) - enum mlx5_inline_modes { MLX5_INLINE_MODE_NONE, MLX5_INLINE_MODE_L2, @@ -172,7 +57,7 @@ enum mlx5_inline_modes { enum { MLX5_MAX_COMMANDS = 32, MLX5_CMD_DATA_BLOCK_SIZE = 512, - MLX5_PCI_CMD_XPORT = 7, + MLX5_PCI_CMD_XPORT = MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT, MLX5_MKEY_BSF_OCTO_SIZE = 4, MLX5_MAX_PSVS = 4, }; @@ -308,63 +193,6 @@ enum { MLX5_EVENT_QUEUE_TYPE_DCT = 6, }; -/* mlx5 components can subscribe to any one of these events via - * mlx5_eq_notifier_register API. - */ -enum mlx5_event { - /* Special value to subscribe to any event */ - MLX5_EVENT_TYPE_NOTIFY_ANY = 0x0, - /* HW events enum start: comp events are not subscribable */ - MLX5_EVENT_TYPE_COMP = 0x0, - /* HW Async events enum start: subscribable events */ - MLX5_EVENT_TYPE_PATH_MIG = 0x01, - MLX5_EVENT_TYPE_COMM_EST = 0x02, - MLX5_EVENT_TYPE_SQ_DRAINED = 0x03, - MLX5_EVENT_TYPE_SRQ_LAST_WQE = 0x13, - MLX5_EVENT_TYPE_SRQ_RQ_LIMIT = 0x14, - - MLX5_EVENT_TYPE_CQ_ERROR = 0x04, - MLX5_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, - MLX5_EVENT_TYPE_PATH_MIG_FAILED = 0x07, - MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, - MLX5_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, - MLX5_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, - MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27, - - MLX5_EVENT_TYPE_INTERNAL_ERROR = 0x08, - MLX5_EVENT_TYPE_PORT_CHANGE = 0x09, - MLX5_EVENT_TYPE_GPIO_EVENT = 0x15, - MLX5_EVENT_TYPE_PORT_MODULE_EVENT = 0x16, - MLX5_EVENT_TYPE_TEMP_WARN_EVENT = 0x17, - MLX5_EVENT_TYPE_XRQ_ERROR = 0x18, - MLX5_EVENT_TYPE_REMOTE_CONFIG = 0x19, - MLX5_EVENT_TYPE_GENERAL_EVENT = 0x22, - MLX5_EVENT_TYPE_MONITOR_COUNTER = 0x24, - MLX5_EVENT_TYPE_PPS_EVENT = 0x25, - - MLX5_EVENT_TYPE_DB_BF_CONGESTION = 0x1a, - MLX5_EVENT_TYPE_STALL_EVENT = 0x1b, - - MLX5_EVENT_TYPE_CMD = 0x0a, - MLX5_EVENT_TYPE_PAGE_REQUEST = 0xb, - - MLX5_EVENT_TYPE_PAGE_FAULT = 0xc, - MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0xd, - - MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0xe, - MLX5_EVENT_TYPE_VHCA_STATE_CHANGE = 0xf, - - MLX5_EVENT_TYPE_DCT_DRAINED = 0x1c, - MLX5_EVENT_TYPE_DCT_KEY_VIOLATION = 0x1d, - - MLX5_EVENT_TYPE_FPGA_ERROR = 0x20, - MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21, - - MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26, - - MLX5_EVENT_TYPE_MAX = 0x100, -}; - enum mlx5_driver_event { MLX5_DRIVER_EVENT_TYPE_TRAP = 0, MLX5_DRIVER_EVENT_UPLINK_NETDEV, @@ -420,22 +248,6 @@ enum { }; enum { - MLX5_OPCODE_NOP = 0x00, - MLX5_OPCODE_SEND_INVAL = 0x01, - MLX5_OPCODE_RDMA_WRITE = 0x08, - MLX5_OPCODE_RDMA_WRITE_IMM = 0x09, - MLX5_OPCODE_SEND = 0x0a, - MLX5_OPCODE_SEND_IMM = 0x0b, - MLX5_OPCODE_LSO = 0x0e, - MLX5_OPCODE_RDMA_READ = 0x10, - MLX5_OPCODE_ATOMIC_CS = 0x11, - MLX5_OPCODE_ATOMIC_FA = 0x12, - MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14, - MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15, - MLX5_OPCODE_BIND_MW = 0x18, - MLX5_OPCODE_CONFIG_CMD = 0x1f, - MLX5_OPCODE_ENHANCED_MPSW = 0x29, - MLX5_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, MLX5_RECV_OPCODE_SEND = 0x01, MLX5_RECV_OPCODE_SEND_IMM = 0x02, @@ -443,19 +255,6 @@ enum { MLX5_CQE_OPCODE_ERROR = 0x1e, MLX5_CQE_OPCODE_RESIZE = 0x16, - - MLX5_OPCODE_SET_PSV = 0x20, - MLX5_OPCODE_GET_PSV = 0x21, - MLX5_OPCODE_CHECK_PSV = 0x22, - MLX5_OPCODE_DUMP = 0x23, - MLX5_OPCODE_RGET_PSV = 0x26, - MLX5_OPCODE_RCHECK_PSV = 0x27, - - MLX5_OPCODE_UMR = 0x25, - - MLX5_OPCODE_FLOW_TBL_ACCESS = 0x2c, - - MLX5_OPCODE_ACCESS_ASO = 0x2d, }; enum { @@ -1224,12 +1023,6 @@ enum mlx5_flex_parser_protos { /* MLX5 DEV CAPs */ -/* TODO: EAT.ME */ -enum mlx5_cap_mode { - HCA_CAP_OPMOD_GET_MAX = 0, - HCA_CAP_OPMOD_GET_CUR = 1, -}; - /* Any new cap addition must update mlx5_hca_caps_alloc() to allocate * capability memory. */ @@ -1508,26 +1301,6 @@ enum mlx5_qcam_feature_groups { MLX5_GET(psp_cap, (mdev)->caps.hca[MLX5_CAP_PSP]->cur, cap) enum { - MLX5_CMD_STAT_OK = 0x0, - MLX5_CMD_STAT_INT_ERR = 0x1, - MLX5_CMD_STAT_BAD_OP_ERR = 0x2, - MLX5_CMD_STAT_BAD_PARAM_ERR = 0x3, - MLX5_CMD_STAT_BAD_SYS_STATE_ERR = 0x4, - MLX5_CMD_STAT_BAD_RES_ERR = 0x5, - MLX5_CMD_STAT_RES_BUSY = 0x6, - MLX5_CMD_STAT_NOT_READY = 0x7, - MLX5_CMD_STAT_LIM_ERR = 0x8, - MLX5_CMD_STAT_BAD_RES_STATE_ERR = 0x9, - MLX5_CMD_STAT_IX_ERR = 0xa, - MLX5_CMD_STAT_NO_RES_ERR = 0xf, - MLX5_CMD_STAT_BAD_INP_LEN_ERR = 0x50, - MLX5_CMD_STAT_BAD_OUTP_LEN_ERR = 0x51, - MLX5_CMD_STAT_BAD_QP_STATE_ERR = 0x10, - MLX5_CMD_STAT_BAD_PKT_ERR = 0x30, - MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR = 0x40, -}; - -enum { MLX5_IEEE_802_3_COUNTERS_GROUP = 0x0, MLX5_RFC_2863_COUNTERS_GROUP = 0x1, MLX5_RFC_2819_COUNTERS_GROUP = 0x2,
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index a5fb747..697fc93 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h
@@ -5986,6 +5986,108 @@ struct mlx5_ifc_cqe_error_syndrome_bits { u8 syndrome[0x8]; }; +struct mlx5_ifc_wqe_ctrl_seg_bits { + u8 opmod[0x8]; + u8 wqe_index[0x10]; + u8 opcode[0x8]; + + u8 qp_or_sq[0x18]; + u8 reserved_at_38[0x2]; + u8 ds[0x6]; + + u8 signature[0x8]; + u8 reserved_at_48[0x10]; + u8 fm[0x3]; + u8 reserved_at_5b[0x1]; + u8 ce[0x2]; + u8 se[0x1]; + u8 reserved_at_5f[0x1]; + + u8 imm[0x20]; +}; + +/* Values for wqe_ctrl_seg_bits.opcode */ +enum { + MLX5_OPCODE_NOP = 0x00, + MLX5_OPCODE_SEND_INVAL = 0x01, + MLX5_OPCODE_RDMA_WRITE = 0x08, + MLX5_OPCODE_RDMA_WRITE_IMM = 0x09, + MLX5_OPCODE_SEND = 0x0a, + MLX5_OPCODE_SEND_IMM = 0x0b, + MLX5_OPCODE_LSO = 0x0e, + MLX5_OPCODE_RDMA_READ = 0x10, + MLX5_OPCODE_ATOMIC_CS = 0x11, + MLX5_OPCODE_ATOMIC_FA = 0x12, + MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14, + MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15, + MLX5_OPCODE_BIND_MW = 0x18, + MLX5_OPCODE_CONFIG_CMD = 0x1f, + MLX5_OPCODE_SET_PSV = 0x20, + MLX5_OPCODE_GET_PSV = 0x21, + MLX5_OPCODE_CHECK_PSV = 0x22, + MLX5_OPCODE_DUMP = 0x23, + MLX5_OPCODE_UMR = 0x25, + MLX5_OPCODE_RGET_PSV = 0x26, + MLX5_OPCODE_RCHECK_PSV = 0x27, + MLX5_OPCODE_ENHANCED_MPSW = 0x29, + MLX5_OPCODE_FLOW_TBL_ACCESS = 0x2c, + MLX5_OPCODE_ACCESS_ASO = 0x2d, +}; + +/* Values for wqe_ctrl_seg_bits.ce */ +enum { + MLX5_WQE_CE_CQE_ALWAYS = 2, +}; + +struct mlx5_ifc_wqe_raddr_seg_bits { + u8 raddr[0x40]; + + u8 rkey[0x20]; + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_wqe_data_seg_bits { + u8 reserved_at_0[0x1]; + u8 byte_count[0x1f]; + + u8 lkey[0x20]; + + u8 addr[0x40]; +}; + +struct mlx5_ifc_cqe64_bits { + u8 reserved_at_0[0x1a0]; + + union { + u8 reserved_at_1a0[0x20]; + struct mlx5_ifc_cqe_error_syndrome_bits error_syndrome; + }; + + u8 send_wqe_opcode[0x8]; + u8 qpn_or_dctn_or_flow_tag[0x18]; + + u8 wqe_counter[0x10]; + u8 signature[0x8]; + u8 opcode[0x4]; + u8 cqe_format[0x2]; + u8 se[0x1]; + u8 owner[0x1]; +}; + +/* Values for cqe64_bits.opcode */ +enum { + MLX5_CQE_REQ = 0, + MLX5_CQE_RESP_WR_IMM = 1, + MLX5_CQE_RESP_SEND = 2, + MLX5_CQE_RESP_SEND_IMM = 3, + MLX5_CQE_RESP_SEND_INV = 4, + MLX5_CQE_RESIZE_CQ = 5, + MLX5_CQE_SIG_ERR = 12, + MLX5_CQE_REQ_ERR = 13, + MLX5_CQE_RESP_ERR = 14, + MLX5_CQE_INVALID = 15, +}; + struct mlx5_ifc_qp_context_extension_bits { u8 reserved_at_0[0x60]; @@ -6482,6 +6584,12 @@ struct mlx5_ifc_query_hca_cap_in_bits { u8 reserved_at_60[0x20]; }; +/* Values for query_hca_cap_in_bits.op_mod */ +enum mlx5_cap_mode { + HCA_CAP_OPMOD_GET_MAX = 0, + HCA_CAP_OPMOD_GET_CUR = 1, +}; + struct mlx5_ifc_other_hca_cap_bits { u8 roce[0x1]; u8 reserved_at_1[0x27f]; @@ -11280,6 +11388,55 @@ struct mlx5_ifc_eqe_bits { u8 owner[0x1]; }; +/* Values for eqe_bits.event_type */ +enum mlx5_event { + /* Special value to subscribe to any event */ + MLX5_EVENT_TYPE_NOTIFY_ANY = 0x0, + /* HW events enum start: comp events are not subscribable */ + MLX5_EVENT_TYPE_COMP = 0x0, + /* HW Async events enum start: subscribable events */ + MLX5_EVENT_TYPE_PATH_MIG = 0x01, + MLX5_EVENT_TYPE_COMM_EST = 0x02, + MLX5_EVENT_TYPE_SQ_DRAINED = 0x03, + MLX5_EVENT_TYPE_SRQ_LAST_WQE = 0x13, + MLX5_EVENT_TYPE_SRQ_RQ_LIMIT = 0x14, + + MLX5_EVENT_TYPE_CQ_ERROR = 0x04, + MLX5_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + MLX5_EVENT_TYPE_PATH_MIG_FAILED = 0x07, + MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, + MLX5_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, + MLX5_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, + MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27, + + MLX5_EVENT_TYPE_INTERNAL_ERROR = 0x08, + MLX5_EVENT_TYPE_PORT_CHANGE = 0x09, + MLX5_EVENT_TYPE_CMD = 0x0a, + MLX5_EVENT_TYPE_PAGE_REQUEST = 0x0b, + MLX5_EVENT_TYPE_PAGE_FAULT = 0x0c, + MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0x0d, + MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0x0e, + MLX5_EVENT_TYPE_VHCA_STATE_CHANGE = 0x0f, + MLX5_EVENT_TYPE_GPIO_EVENT = 0x15, + MLX5_EVENT_TYPE_PORT_MODULE_EVENT = 0x16, + MLX5_EVENT_TYPE_TEMP_WARN_EVENT = 0x17, + MLX5_EVENT_TYPE_XRQ_ERROR = 0x18, + MLX5_EVENT_TYPE_REMOTE_CONFIG = 0x19, + MLX5_EVENT_TYPE_DB_BF_CONGESTION = 0x1a, + MLX5_EVENT_TYPE_STALL_EVENT = 0x1b, + MLX5_EVENT_TYPE_DCT_DRAINED = 0x1c, + MLX5_EVENT_TYPE_DCT_KEY_VIOLATION = 0x1d, + MLX5_EVENT_TYPE_FPGA_ERROR = 0x20, + MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21, + MLX5_EVENT_TYPE_GENERAL_EVENT = 0x22, + MLX5_EVENT_TYPE_MONITOR_COUNTER = 0x24, + MLX5_EVENT_TYPE_PPS_EVENT = 0x25, + MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26, + + MLX5_EVENT_TYPE_MAX = 0x100, +}; + +/* Values for cmd_queue_entry_bits.type */ enum { MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT = 0x7, }; @@ -11322,6 +11479,27 @@ struct mlx5_ifc_cmd_out_bits { u8 command_output[0x20]; }; +/* Values for cmd_out_bits.status */ +enum { + MLX5_CMD_STAT_OK = 0x0, + MLX5_CMD_STAT_INT_ERR = 0x1, + MLX5_CMD_STAT_BAD_OP_ERR = 0x2, + MLX5_CMD_STAT_BAD_PARAM_ERR = 0x3, + MLX5_CMD_STAT_BAD_SYS_STATE_ERR = 0x4, + MLX5_CMD_STAT_BAD_RES_ERR = 0x5, + MLX5_CMD_STAT_RES_BUSY = 0x6, + MLX5_CMD_STAT_NOT_READY = 0x7, + MLX5_CMD_STAT_LIM_ERR = 0x8, + MLX5_CMD_STAT_BAD_RES_STATE_ERR = 0x9, + MLX5_CMD_STAT_IX_ERR = 0xa, + MLX5_CMD_STAT_NO_RES_ERR = 0xf, + MLX5_CMD_STAT_BAD_QP_STATE_ERR = 0x10, + MLX5_CMD_STAT_BAD_PKT_ERR = 0x30, + MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR = 0x40, + MLX5_CMD_STAT_BAD_INP_LEN_ERR = 0x50, + MLX5_CMD_STAT_BAD_OUTP_LEN_ERR = 0x51, +}; + struct mlx5_ifc_cmd_in_bits { u8 opcode[0x10]; u8 reserved_at_10[0x10];
diff --git a/include/linux/mlx5/mlx5_ifc_macros.h b/include/linux/mlx5/mlx5_ifc_macros.h new file mode 100644 index 0000000..1b9e39a50 --- /dev/null +++ b/include/linux/mlx5/mlx5_ifc_macros.h
@@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2013-2026, Mellanox Technologies. All rights reserved. + * + * Accessor macros for mlx5 IFC structures. + * + * Extracted from device.h so that code which cannot include device.h + * (e.g. selftests) can still use the MLX5_SET/GET family directly. + */ + +#ifndef MLX5_IFC_MACROS_H +#define MLX5_IFC_MACROS_H + +/* Internal helpers -- 32-bit */ +#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0) +#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld) +#define __mlx5_bit_off(typ, fld) (offsetof(struct mlx5_ifc_##typ##_bits, fld)) +#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16) +#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32) +#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64) +#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0xf)) +#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f)) +#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) +#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld)) +#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) +#define __mlx5_16_mask(typ, fld) (__mlx5_mask16(typ, fld) << __mlx5_16_bit_off(typ, fld)) +#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits) + +/* Size and address macros */ +#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8) +#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8) +#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32) +#define MLX5_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64) +#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8) +#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32) +#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8) +#define MLX5_ADDR_OF(typ, p, fld) ((void *)((u8 *)(p) + MLX5_BYTE_OFF(typ, fld))) + +/* insert a value to a struct */ +#define MLX5_SET(typ, p, fld, v) do { \ + u32 _v = v; \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ + *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ + cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ + (~__mlx5_dw_mask(typ, fld))) | (((_v) & __mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld))); \ +} while (0) + +#define MLX5_ARRAY_SET(typ, p, fld, idx, v) do { \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 32); \ + MLX5_SET(typ, p, fld[idx], v); \ +} while (0) + +#define MLX5_SET_TO_ONES(typ, p, fld) do { \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ + *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ + cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ + (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld))); \ +} while (0) + +#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\ +__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \ +__mlx5_mask(typ, fld)) + +#define MLX5_GET_PR(typ, p, fld) ({ \ + u32 ___t = MLX5_GET(typ, p, fld); \ + pr_debug(#fld " = 0x%x\n", ___t); \ + ___t; \ +}) + +/* 64-bit field accessors */ +#define __MLX5_SET64(typ, p, fld, v) do { \ + BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \ + *((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \ +} while (0) + +#define MLX5_SET64(typ, p, fld, v) do { \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ + __MLX5_SET64(typ, p, fld, v); \ +} while (0) + +#define MLX5_ARRAY_SET64(typ, p, fld, idx, v) do { \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ + __MLX5_SET64(typ, p, fld[idx], v); \ +} while (0) + +#define MLX5_ARRAY_GET64(typ, p, fld, idx) \ + ({ \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ + be64_to_cpu( \ + *((__be64 *)(p) + __mlx5_64_off(typ, fld) + (idx))); \ + }) + +#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld))) + +#define MLX5_GET64_PR(typ, p, fld) ({ \ + u64 ___t = MLX5_GET64(typ, p, fld); \ + pr_debug(#fld " = 0x%llx\n", ___t); \ + ___t; \ +}) + +/* 16-bit field accessors */ +#define MLX5_GET16(typ, p, fld) ((be16_to_cpu(*((__be16 *)(p) +\ +__mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \ +__mlx5_mask16(typ, fld)) + +#define MLX5_SET16(typ, p, fld, v) do { \ + u16 _v = v; \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 16); \ + *((__be16 *)(p) + __mlx5_16_off(typ, fld)) = \ + cpu_to_be16((be16_to_cpu(*((__be16 *)(p) + __mlx5_16_off(typ, fld))) & \ + (~__mlx5_16_mask(typ, fld))) | (((_v) & __mlx5_mask16(typ, fld)) \ + << __mlx5_16_bit_off(typ, fld))); \ +} while (0) + +/* Big endian getters */ +#define MLX5_GET64_BE(typ, p, fld) (*((__be64 *)(p) +\ + __mlx5_64_off(typ, fld))) + +#define MLX5_GET_BE(type_t, typ, p, fld) ({ \ + type_t tmp; \ + switch (sizeof(tmp)) { \ + case sizeof(u8): \ + tmp = (__force type_t)MLX5_GET(typ, p, fld); \ + break; \ + case sizeof(u16): \ + tmp = (__force type_t)cpu_to_be16(MLX5_GET(typ, p, fld)); \ + break; \ + case sizeof(u32): \ + tmp = (__force type_t)cpu_to_be32(MLX5_GET(typ, p, fld)); \ + break; \ + case sizeof(u64): \ + tmp = (__force type_t)MLX5_GET64_BE(typ, p, fld); \ + break; \ + } \ + tmp; \ + }) + +/* + * Use READ_ONCE/WRITE_ONCE for a single field that hardware may read/write + * unpredictably, mostly owner bits. All other bits in the DW must be stable. + * Usually a dma_wmb() will be required before a write and a dma_rmb() after a + * read. + */ +#define MLX5_GET_ONCE(typ, p, fld) \ + ((be32_to_cpu(READ_ONCE(*((__be32 *)(p) + __mlx5_dw_off(typ, fld)))) >> \ + __mlx5_dw_bit_off(typ, fld)) & \ + __mlx5_mask(typ, fld)) + +#define MLX5_SET_ONCE(typ, p, fld, v) \ + do { \ + u32 _v = v; \ + __be32 *_dw = (__be32 *)(p) + __mlx5_dw_off(typ, fld); \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ + WRITE_ONCE(*_dw, \ + cpu_to_be32((be32_to_cpu(READ_ONCE(*_dw)) & \ + (~__mlx5_dw_mask(typ, fld))) | \ + (((_v) & __mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld)))); \ + } while (0) + +/* Access MMIO registers, usually the init segment, using IFC structs. */ +#define MLX5_GET_MMIO(typ, p, fld) \ + ((ioread32be(((__be32 __iomem *)(p) + __mlx5_dw_off(typ, fld))) >> \ + __mlx5_dw_bit_off(typ, fld)) & \ + __mlx5_mask(typ, fld)) + +/* The set is not relaxed so there is an integrated dma_wmb(). */ +#define MLX5_SET_MMIO(typ, p, fld, v) \ + do { \ + u32 _v = v; \ + void __iomem *_dw = \ + ((__be32 __iomem *)(p) + __mlx5_dw_off(typ, fld)); \ + if (__mlx5_bit_sz(typ, fld) == 32) \ + iowrite32be(_v, _dw); \ + else \ + iowrite32be((ioread32be(_dw) & \ + (~__mlx5_dw_mask(typ, fld))) | \ + ((_v & __mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld)), \ + _dw); \ + } while (0) + +#endif /* MLX5_IFC_MACROS_H */
diff --git a/include/linux/mm.h b/include/linux/mm.h index 0a2a7fc..6882ad4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h
@@ -3314,10 +3314,10 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); -#ifdef CONFIG_BPF_SYSCALL -extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags); -#endif +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages,
diff --git a/include/linux/module.h b/include/linux/module.h index 2c5a026..76f4f35 100644 --- a/include/linux/module.h +++ b/include/linux/module.h
@@ -503,6 +503,10 @@ struct module { void *btf_data; void *btf_base_data; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + unsigned int btf_inline_data_size; + void *btf_inline_data; +#endif #ifdef CONFIG_JUMP_LABEL struct jump_entry *jump_entries; unsigned int num_jump_entries;
diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index a443687..ff0ad1e 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h
@@ -15,6 +15,7 @@ #include <linux/cdev.h> #include <linux/completion.h> #include <linux/idr.h> +#include <linux/srcu.h> #include <linux/of.h> #include <linux/rsc_table.h> @@ -228,6 +229,7 @@ enum rproc_features { * @mappings: list of iommu mappings we initiated, needed on shutdown * @bootaddr: address of first instruction to boot rproc with (optional) * @rvdevs: list of remote virtio devices + * @vq_srcu: SRCU domain for the virtqueue callbacks of @rvdevs * @subdevs: list of subdevices, to following the running state * @notifyids: idr for dynamically assigning rproc-wide unique notify ids * @index: index of this rproc device @@ -274,6 +276,7 @@ struct rproc { struct list_head mappings; u64 bootaddr; struct list_head rvdevs; + struct srcu_struct vq_srcu; struct list_head subdevs; struct idr notifyids; int index;
diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index eac3e90..e437006 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h
@@ -99,6 +99,13 @@ struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flag void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size); /* + * A minimum of 2 pages ensures we can always swap a page out of the ring + * buffer. This does not account for the reader page or any other page + * containing meta data used by static ring buffers. + */ +#define RING_BUFFER_MIN_NR_PAGES 2UL + +/* * Because the ring buffer is generic, if other users of the ring buffer get * traced by ftrace, it can produce lockdep warnings. We need to keep each * ring buffer's lock class separate. @@ -287,7 +294,7 @@ static inline struct ring_buffer_desc *__first_ring_buffer_desc(struct trace_buf static inline unsigned long __calc_nr_pages_ring_buffer_desc(size_t size) { /* Takes into account the reader page */ - return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), 2UL) + 1; + return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), RING_BUFFER_MIN_NR_PAGES) + 1; } static inline size_t trace_buffer_desc_size(size_t buffer_size, unsigned int nr_cpus)
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 2a42124..5f24edf4 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h
@@ -38,7 +38,7 @@ struct scatterlist; #define IO_TLB_DEFAULT_SIZE (64UL << 20) #endif -unsigned long swiotlb_size_or_default(void); +unsigned long swiotlb_default_pool_size(void); void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, int (*remap)(void *tlb, unsigned long nslabs)); int swiotlb_init_late(size_t size, gfp_t gfp_mask,
diff --git a/include/linux/tcp.h b/include/linux/tcp.h index e4d1720..d18f7bc 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h
@@ -233,6 +233,13 @@ struct tcp_sock { is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ +#ifdef CONFIG_BPF + u32 bpf_tcp_ops_flags; +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) \ + (READ_ONCE((TP)->bpf_tcp_ops_flags) & BPF_TCP_OPS_FLAG_ ## ARG) +#else +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) (0) +#endif __cacheline_group_end(tcp_sock_read_txrx); /* RX read-mostly hotpath cache lines */
diff --git a/include/linux/tnum.h b/include/linux/tnum.h index ca2cfec..1668224 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h
@@ -134,4 +134,6 @@ static inline bool tnum_subreg_is_const(struct tnum a) /* Returns the smallest member of t larger than z */ u64 tnum_step(struct tnum t, u64 z); +u32 tnum_alignment(struct tnum a); + #endif /* _LINUX_TNUM_H */
diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h index c8ae1e1..e4cc2d4 100644 --- a/include/linux/trace_remote_event.h +++ b/include/linux/trace_remote_event.h
@@ -3,6 +3,8 @@ #ifndef _LINUX_TRACE_REMOTE_EVENTS_H #define _LINUX_TRACE_REMOTE_EVENTS_H +#include <linux/types.h> + struct trace_remote; struct trace_event_fields; struct trace_seq;
diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 45f08986..0cc91c6 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h
@@ -54,7 +54,7 @@ struct vfio_device { struct list_head dev_set_list; unsigned int migration_flags; u8 precopy_info_v2; - struct kvm *kvm; + struct file *kvm; /* Members below here are private, not for driver use */ unsigned int index; @@ -66,7 +66,6 @@ struct vfio_device { unsigned int open_count; struct completion comp; struct iommufd_access *iommufd_access; - void (*put_kvm)(struct kvm *kvm); struct inode *inode; #if IS_ENABLED(CONFIG_IOMMUFD) struct iommufd_device *iommufd_device; @@ -378,7 +377,7 @@ static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *devi #endif bool vfio_file_is_valid(struct file *file); bool vfio_file_enforced_coherent(struct file *file); -void vfio_file_set_kvm(struct file *file, struct kvm *kvm); +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long))
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 9a1674c..2f3fa27 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h
@@ -8,6 +8,7 @@ * Author: Tom Lyon, pugs@cisco.com */ +#include <linux/device.h> #include <linux/mutex.h> #include <linux/pci.h> #include <linux/vfio.h> @@ -16,6 +17,7 @@ #include <linux/types.h> #include <linux/uuid.h> #include <linux/notifier.h> +#include <linux/pm.h> #ifndef VFIO_PCI_CORE_H #define VFIO_PCI_CORE_H @@ -168,7 +170,6 @@ int vfio_pci_core_init_dev(struct vfio_device *core_vdev); void vfio_pci_core_release_dev(struct vfio_device *core_vdev); int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); -extern const struct pci_error_handlers vfio_pci_core_err_handlers; int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, int nr_virtfn); long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, @@ -193,8 +194,6 @@ int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev, int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); -pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, - pci_channel_state_t state); ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, void __iomem *io, char __user *buf, loff_t off, size_t count, size_t x_start, @@ -260,4 +259,40 @@ vfio_pci_core_get_iomap(struct vfio_pci_core_device *vdev, unsigned int bar) int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, struct phys_vec *phys); +int vfio_pci_core_runtime_suspend(struct vfio_pci_core_device *vdev); +int vfio_pci_core_runtime_resume(struct vfio_pci_core_device *vdev); +pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev); + +/* + * Per-driver PM/AER trampolines. driver_data points to the driver's private + * structure, which embeds a struct vfio_pci_core_device. + * + * @name: prefix for <name>_pm_ops and <name>_aer_err_detected + * @drv_type: type of the driver's private structure, including struct + * @member: member containing the struct vfio_pci_core_device + */ +#define VFIO_PCI_CORE_DEFINE_CALLBACKS(name, drv_type, member) \ +static int __maybe_unused name##_pm_suspend(struct device *dev) \ +{ \ + drv_type *drvdata = dev_get_drvdata(dev); \ + \ + return vfio_pci_core_runtime_suspend(&drvdata->member); \ +} \ +static int __maybe_unused name##_pm_resume(struct device *dev) \ +{ \ + drv_type *drvdata = dev_get_drvdata(dev); \ + \ + return vfio_pci_core_runtime_resume(&drvdata->member); \ +} \ +static const struct dev_pm_ops name##_pm_ops = { \ + SET_RUNTIME_PM_OPS(name##_pm_suspend, name##_pm_resume, NULL) \ +}; \ +static pci_ers_result_t name##_aer_err_detected(struct pci_dev *pdev, \ + pci_channel_state_t state) \ +{ \ + drv_type *drvdata = pci_get_drvdata(pdev); \ + \ + return vfio_pci_core_aer_err_detected(&drvdata->member); \ +} + #endif /* VFIO_PCI_CORE_H */
diff --git a/include/linux/vgaarb.h b/include/linux/vgaarb.h index 97129a1..71a3646 100644 --- a/include/linux/vgaarb.h +++ b/include/linux/vgaarb.h
@@ -33,7 +33,8 @@ struct pci_dev *vga_default_device(void); void vga_set_default_device(struct pci_dev *pdev); int vga_remove_vgacon(struct pci_dev *pdev); int vga_client_register(struct pci_dev *pdev, - unsigned int (*set_decode)(struct pci_dev *pdev, bool state)); + unsigned int (*set_decode)(void *data, bool state), + void *data); #else /* CONFIG_VGA_ARB */ static inline void vga_set_legacy_decoding(struct pci_dev *pdev, unsigned int decodes) @@ -59,7 +60,8 @@ static inline int vga_remove_vgacon(struct pci_dev *pdev) return 0; } static inline int vga_client_register(struct pci_dev *pdev, - unsigned int (*set_decode)(struct pci_dev *pdev, bool state)) + unsigned int (*set_decode)(void *data, bool state), + void *data) { return 0; } @@ -97,7 +99,7 @@ static inline int vga_get_uninterruptible(struct pci_dev *pdev, static inline void vga_client_unregister(struct pci_dev *pdev) { - vga_client_register(pdev, NULL); + vga_client_register(pdev, NULL, NULL); } #endif /* LINUX_VGA_H */
diff --git a/include/net/tcp.h b/include/net/tcp.h index 0265c27a..3560add 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h
@@ -512,6 +512,7 @@ void tcp_set_keepalive(struct sock *sk, int val); void tcp_syn_ack_timeout(const struct request_sock *req); int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags); +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup); int tcp_set_rcvlowat(struct sock *sk, int val); void tcp_set_rcvbuf(struct sock *sk, int val); int tcp_set_window_clamp(struct sock *sk, int val); @@ -2930,7 +2931,7 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, * program loaded). */ #ifdef CONFIG_BPF -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) { struct bpf_sock_ops_kern sock_ops; int ret; @@ -2947,7 +2948,7 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) if (nargs > 0) memcpy(sock_ops.args, args, nargs * sizeof(*args)); - ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); if (ret == 0) ret = sock_ops.reply; else @@ -2955,42 +2956,37 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) return ret; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - u32 args[2] = {arg1, arg2}; +#define tcp_call_bpf(sk, op) \ +({ \ + int __ret = 0; \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) { \ + __ret = __tcp_call_bpf(sk, op, 0, NULL); \ + } \ + __ret; \ +}) - return tcp_call_bpf(sk, op, 2, args); -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - u32 args[3] = {arg1, arg2, arg3}; - - return tcp_call_bpf(sk, op, 3, args); -} +#define tcp_call_bpf_flag(sk, op, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && \ + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) { \ + u32 __args[] = { __VA_ARGS__ }; \ + __tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args); \ + } \ +} while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; + WRITE_ONCE(tcp_sk(sk)->bpf_tcp_ops_flags, 0); } #else -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int tcp_call_bpf(struct sock *sk, int op) { return -EPERM; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - return -EPERM; -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - return -EPERM; -} +#define tcp_call_bpf_flag(sk, op, ...) do { } while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { @@ -2998,12 +2994,196 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) #endif +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF) + +struct bpf_tcp_ops { + /* Should return the initial SYN (active open) or SYN-ACK (passive open) + * retransmission timeout. Return the timeout in jiffies, or <= 0 for + * the kernel default. + * + * @req: request_sock on the passive (synack) path; NULL otherwise. + */ + int (*timeout_init)(struct sock *sk, struct request_sock *req); + + /* Should return the initial advertised receive window, in packets, + * or < 0 for the kernel default. @req as in timeout_init(). + */ + int (*rwnd_init)(struct sock *sk, struct request_sock *req); + + /* Called when an active connection becomes established. + * @skb is the SYNACK that completed the 3WHS, or NULL for a + * TCP_REPAIR socket (tcp_finish_connect() with no skb). + */ + void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable); + + /* Called when a passive connection becomes established. + * @skb is the ACK that completed the 3WHS. + */ + void (*passive_established)(struct sock *sk, struct sk_buff *skb); + + /* Called when the retransmission timer fires. */ + void (*rto)(struct sock *sk); + + /* Called on every RTT sample if BPF_TCP_OPS_FLAG_RTT is enabled. + * @mrtt: the measured RTT, in microseconds. + * @srtt: the updated smoothed RTT. + */ + void (*rtt)(struct sock *sk, long mrtt, u32 srtt); + + /* Called when the connection changes TCP state. + * @state: the new state (one of the TCP_* states). + */ + void (*set_state)(struct sock *sk, int state); + + /* Called when an skb is retransmitted. + * @skb: the retransmitted skb. + * @err: tcp_transmit_skb() return value (0 on success). + */ + void (*retrans)(struct sock *sk, struct sk_buff *skb, int err); + + /* Called right before an active connection is initialized. */ + void (*connect)(struct sock *sk); + + /* Called on listen(2), right after the socket enters TCP_LISTEN. */ + void (*listen)(struct sock *sk); + + /* + * Parse the TCP header options of an incoming skb received on an + * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes() + * to access the options. + * + * Called if BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL is enabled, or if + * BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN is enabled and an unknown + * option is received. + */ + void (*parse_hdr)(struct sock *sk, struct sk_buff *skb); + + /* + * Reserve space in the outgoing TCP header for options to be written + * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes. + * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * + * @skb: outgoing packet. NULL when called from tcp_current_mss() + * (MSS sizing). + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @remaining: pointer to the size of space still available; cast it + * using bpf_rdonly_cast() before dereferencing. + */ + void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + unsigned int *remaining); + + /* + * Write header options into the space reserved earlier by hdr_opt_len(). + * Use bpf_store_hdr_opt() to write; it appends within the reserved window + * shared with legacy SOCKOPS. + * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * + * @skb: outgoing packet. + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @opt_off: offset in the outgoing @skb's TCP header where the + * bpf_tcp_ops portion of the reserved window begins, i.e. after + * the kernel and legacy options. + */ + void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + u32 opt_off); + + /* + * Called when an incoming skb is enqueued to sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*enqueue_rcvq)(struct sock *sk, struct sk_buff *skb); + + /* + * Called after data is dequeued from sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*dequeue_rcvq)(struct sock *sk); +}; + +#define __bpf_tcp_ops_call(op, sk, ...) \ +do { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct cgroup *cgrp; \ + \ + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ + rcu_read_lock_dont_migrate(); \ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + tcp_ops->op(sk, ##__VA_ARGS__); \ + } \ + rcu_read_unlock_migrate(); \ +} while (0) + +#define bpf_tcp_ops_call(op, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) && \ + BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \ +({ \ + int __retval = (init_retval); \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct bpf_tramp_run_ctx run_ctx; \ + struct bpf_run_ctx *old_run_ctx; \ + struct sock *__sk = sk_to_full_sk(sk); \ + struct request_sock *req = NULL; \ + struct cgroup *cgrp; \ + \ + if (__sk) { \ + run_ctx.retval = (init_retval); \ + cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data); \ + if (!sk_fullsock(sk)) \ + req = (struct request_sock *)sk; \ + rcu_read_lock_dont_migrate(); \ + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \ + } \ + bpf_reset_run_ctx(old_run_ctx); \ + rcu_read_unlock_migrate(); \ + __retval = run_ctx.retval; \ + } \ + } \ + __retval; \ +}) + +#else +#define __bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) +#endif + static inline u32 tcp_timeout_init(struct sock *sk) { int timeout; - timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); - + timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT); + timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk); if (timeout <= 0) timeout = TCP_TIMEOUT_INIT; return min_t(int, timeout, TCP_RTO_MAX); @@ -3013,8 +3193,8 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) { int rwnd; - rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); - + rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT); + rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk); if (rwnd < 0) rwnd = 0; return rwnd; @@ -3022,13 +3202,13 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) { - return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); + return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1); } static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) { - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt); } #if IS_ENABLED(CONFIG_SMC)
diff --git a/include/trace/define_remote_events.h b/include/trace/define_remote_events.h index 676e803..78346ac 100644 --- a/include/trace/define_remote_events.h +++ b/include/trace/define_remote_events.h
@@ -35,17 +35,26 @@ do { \ #define RE_PRINTK(__args...) __args -#define REMOTE_EVENT(__name, __id, __struct, __printk) \ - REMOTE_EVENT_FORMAT(__name, __struct); \ +#define REMOTE_EVENT_PRINT_FUNC(__name, __printk) \ static void remote_event_print_##__name(void *evt, struct trace_seq *seq) \ { \ struct remote_event_format_##__name __maybe_unused *__entry = evt; \ trace_seq_puts(seq, #__name); \ - remote_printk(__printk); \ + __printk; \ } + +#define REMOTE_EVENT(__name, __id, __struct, __printk) \ + REMOTE_EVENT_FORMAT(__name, __struct); \ + REMOTE_EVENT_PRINT_FUNC(__name, remote_printk(__printk)) + +#define REMOTE_EVENT_CUSTOM_PRINTK(__name, __id, __struct, __printk) \ + REMOTE_EVENT_FORMAT(__name, __struct); \ + REMOTE_EVENT_PRINT_FUNC(__name, __printk) + #include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE) #undef REMOTE_EVENT +#undef REMOTE_EVENT_CUSTOM_PRINTK #undef RE_PRINTK #undef re_field #define re_field(__type, __field) \ @@ -70,4 +79,8 @@ do { \ .print_fmt = remote_event_print_fmt_##__name, \ .print = remote_event_print_##__name, \ } + +#define REMOTE_EVENT_CUSTOM_PRINTK(__name, __id, __struct, __printk) \ + REMOTE_EVENT(__name, __id, RE_STRUCT(__struct), "\"Unknown\"") + #include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE)
diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index b282e3a..07013f8 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h
@@ -4,8 +4,14 @@ #include <linux/tracepoint.h> +#undef UNPACK_KVM +#define UNPACK_KVM(name) name #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM UNPACK_KVM(KVM_DEV_NAME) +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR kvm +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE kvm #define ERSN(x) { KVM_EXIT_##x, "KVM_EXIT_" #x } @@ -358,13 +364,13 @@ TRACE_EVENT(kvm_dirty_ring_exit, TP_printk("vcpu %d", __entry->vcpu_id) ); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* * @start: Starting address of guest memory range * @end: End address of guest memory range * @attr: The value of the attribute being set. */ -TRACE_EVENT(kvm_vm_set_mem_attributes, +TRACE_EVENT(kvm_set_vm_mem_attributes, TP_PROTO(gfn_t start, gfn_t end, unsigned long attr), TP_ARGS(start, end, attr), @@ -383,7 +389,7 @@ TRACE_EVENT(kvm_vm_set_mem_attributes, TP_printk("%#016llx -- %#016llx [0x%lx]", __entry->start, __entry->end, __entry->attr) ); -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ TRACE_EVENT(kvm_unmap_hva_range, TP_PROTO(unsigned long start, unsigned long end),
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 732b35c..f29b0aa 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h
@@ -56,6 +56,19 @@ enum bpf_cond_pseudo_jmp { BPF_MAY_GOTO = 0, + /* + * Follows a call of a bpf subprog. Taken when the subprog left with + * BPF_EXIT that has src_reg == BPF_UNWIND_EXIT. + */ + BPF_UNWIND_GOTO = 1, +}; + +enum bpf_pseudo_exit { + /* + * When bpf_exit->src_reg == BPF_UNWIND_EXIT, it's the same as bpf_exit, + * except that BPF_UNWIND_GOTO after the call is taken. + */ + BPF_UNWIND_EXIT = 1, }; /* Register numbers */ @@ -936,6 +949,33 @@ union bpf_iter_link_info { * 0 on success or -1 if an error occurred (in which case, * *errno* is set appropriately). * + * BPF_PROG_STREAM_OPEN + * Description + * Open a file descriptor for one of the BPF streams associated + * with the program identified by *prog_fd*. The stream is selected + * by *stream_id*. + * + * The returned file descriptor supports **read**\ (2) and + * **poll**\ (2). Reads block while the stream is empty unless + * **BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking + * read of an empty stream fails with **EAGAIN**. + * + * **poll**\ (2) reports **POLLIN** when data is available and + * **POLLHUP** once the program has been freed, that is, after every + * reference to it, including links and other file descriptors, has + * been dropped. Hangup may lag the final release because program + * teardown is deferred. Buffered data remains readable after + * **POLLHUP** and a read returns zero after all such data has been + * consumed. + * + * The file descriptor is read-only and has the close-on-exec flag + * set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**. + * *flags* may only contain **BPF_F_STREAM_NONBLOCK**. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -993,6 +1033,7 @@ enum bpf_cmd { BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_STREAM_OPEN, __MAX_BPF_CMD, BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */ }; @@ -1316,6 +1357,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ @@ -1524,6 +1571,11 @@ enum { BPF_STREAM_STDERR = 2, }; +/* flags for BPF_PROG_STREAM_OPEN command */ +enum { + BPF_F_STREAM_NONBLOCK = (1U << 0), +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1756,7 +1808,7 @@ union bpf_attr { __u32 prog_cnt; __u32 count; }; - __u32 :32; + __u32 type_id; /* output: per-program attach_flags. * not allowed to be set during effective query. */ @@ -1950,6 +2002,12 @@ union bpf_attr { __u32 flags; } prog_assoc_struct_ops; + struct { + __u32 prog_fd; + __u32 stream_id; + __u32 flags; + } prog_stream_open; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF @@ -4867,15 +4925,18 @@ union bpf_attr { * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) + * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags) * Description * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). + * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). For the + * **bpf_tcp_ops** struct_ops, this helper can be called from the + * **parse_hdr**\ () and **write_hdr_opt**\ () operators. * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. + * If *flags* is 0, it will search the option from the packet + * associated with the current operation. For + * **BPF_PROG_TYPE_SOCK_OPS**, the comment in + * **struct bpf_sock_ops** has details on what skb_data + * contains under different *op*. * * The first byte of the *searchby_res* specifies the * kind that it wants to search. @@ -4908,6 +4969,8 @@ union bpf_attr { * * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the * saved_syn packet or the just-received syn packet. + * Not supported by the **bpf_tcp_ops** struct_ops, which + * rejects all flags. * * Return * > 0 when found, the header option is copied to *searchby_res*. @@ -4928,9 +4991,9 @@ union bpf_attr { * packet. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) + * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags) * Description * Store header option. The data will be copied * from buffer *from* with length *len* to the TCP header. @@ -4946,7 +5009,9 @@ union bpf_attr { * by searching the same option in the outgoing skb. * * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. + * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the + * **write_hdr_opt**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4961,9 +5026,9 @@ union bpf_attr { * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) + * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags) * Description * Reserve *len* bytes for the bpf header option. The * space will be used by **bpf_store_hdr_opt**\ () later in @@ -4973,7 +5038,9 @@ union bpf_attr { * the total number of bytes will be reserved. * * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. + * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the + * **hdr_opt_len**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4983,7 +5050,7 @@ union bpf_attr { * **-ENOSPC** if there is not enough space in the header. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) * Description @@ -6890,6 +6957,8 @@ struct bpf_link_info { } xdp; struct { __u32 map_id; + __u32 :32; + __u64 cgroup_id; } struct_ops; struct { __u32 pf; @@ -7284,6 +7353,31 @@ enum { */ }; +/* + * Most callbacks of struct bpf_tcp_ops are placed in the slow + * path (e.g., one-shot connection setup or unlikely events like + * timers) and are invoked simply by defining non-NULL callbacks. + * + * Callbacks in the fast path, however, would incur noticeable + * overhead even when set to NULL, so they are disabled by default + * and must be explicitly enabled per socket via bpf_tcp_ops_set_flags(). + * + * The flags are per-socket and shared by all effective bpf_tcp_ops + * programs. + */ +enum { + /* .rtt() */ + BPF_TCP_OPS_FLAG_RTT = (1 << 0), + /* .parse_hdr() */ + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL = (1 << 1), + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), + /* .hdr_opt_len() and .write_hdr_opt() */ + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), + /* .enqueue_rcvq() and .dequeue_rcvq() */ + BPF_TCP_OPS_FLAG_RCVQ = (1 << 4), + BPF_TCP_OPS_FLAG_ALL = (1 << 5) - 1, +}; + /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect * changes between the TCP and BPF versions. Ideally this should never happen. * If it does, we need to add code to convert them before calling @@ -7600,6 +7694,10 @@ struct bpf_task_work { __u64 __opaque; } __attribute__((aligned(8))); +struct bpf_rcu_head { + __u64 __opaque[6]; +} __attribute__((aligned(8))); + struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8)));
diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h index 618167c..11d6587 100644 --- a/include/uapi/linux/btf.h +++ b/include/uapi/linux/btf.h
@@ -54,8 +54,8 @@ struct btf_type { * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64 + * and LOC_PARAM. "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. @@ -92,7 +92,9 @@ enum { BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - + BTF_KIND_LOC_PARAM = 20, /* Location parameter information */ + BTF_KIND_LOC_PROTO = 21, /* Location prototype for site */ + BTF_KIND_LOCSEC = 22, /* Location section */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, }; @@ -212,4 +214,79 @@ struct btf_enum64 { __u32 val_hi32; }; +/* + * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param" + * that contains flags specifying the contents of the vlen-specified + * number of 4-byte values that follow. + */ +struct btf_loc_param { + __u32 flags; + __u32 values[]; +}; + +/* + * The combination of size, vlen and flags gives us the means to interpret + * the following vlen-specified set of 4-byte values: + * + * - a BTF_LOC_PARAM_CONST is a constant value; combination + * of size, vlen and _SIGNED flag determines it. If the value requires + * 64 bits it is stored in {lo,hi} order. + * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be + * normalized with respect to kernel/module base address. + * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number; + * with vlen 2 it is a multi-register parameter. Register numbers are + * numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since + * DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer + * to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for + * the frame base register to avoid collisions. + * - a _REG | OFFSET describes an address without dereferencing it. + * - a _REG | DEREF with vlen 1 dereferences the value in the register + * number specified. + * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number + * in the first 4-byte value and the offset in the remainder. + * In the case of REG and OFFSET combinations, the OFFSET has the width + * of the value words while the type size describes the represented parameter, + * so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be + * an 8-byte register dereference with signed 4-byte offset, since the vlen 2 + * values consist of a register value and the signed value. + */ +enum btf_loc_param_flags { + BTF_LOC_PARAM_SIGNED = 0x1, + BTF_LOC_PARAM_CONST = 0x2, + BTF_LOC_PARAM_ADDR = 0x4, + BTF_LOC_PARAM_REG = 0x8, + BTF_LOC_PARAM_DEREF = 0x10, + BTF_LOC_PARAM_OFFSET = 0x20, +}; + +enum { + BTF_LOC_PARAM_FBREG = 0xffffffff +}; + +/* + * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate + * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a + * vlen-specified number of __u32 BTF type ids which specify the associated + * BTF_KIND_LOC_PARAM for each function parameter associated with the + * location. The type should either be 0 (no location info) or point at + * a BTF_KIND_LOC_PARAM. + */ + +/* + * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc" + * containing location site-specific information for a specific ELF section; + * for example locations in ".text" are in a LOCSEC named "inline.text". + * + * - function (func) + * - location prototype type id (loc_proto) + * - address offset (offset) relative to the runtime base address of the + * ELF section associated with the LOCSEC + */ + +struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; +}; + #endif /* _UAPI__LINUX_BTF_H__ */
diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884e..fa85b97 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h
@@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ac2d77d..8dff2fc 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h
@@ -999,6 +999,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_HPAGE_2G 249 #define KVM_CAP_PPC_COMPAT_CAPS 250 #define KVM_CAP_ARM_PMU_V3_STRICT 251 +#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252 struct kvm_irq_routing_irqchip { __u32 irqchip; @@ -1653,6 +1654,21 @@ struct kvm_memory_attributes { __u64 flags; }; +/* Available with KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES */ +#define KVM_SET_MEMORY_ATTRIBUTES2 _IOWR(KVMIO, 0xd2, struct kvm_memory_attributes2) + +struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 error_offset; + __u64 reserved[11]; +}; + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) #define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd)
diff --git a/init/main.c b/init/main.c index 31f2bf5..27f4cb3 100644 --- a/init/main.c +++ b/init/main.c
@@ -278,7 +278,7 @@ static void * __init get_boot_config_from_initrd(size_t *_size) int i; if (!initrd_end || initrd_end < initrd_start || - initrd_end - initrd_start < BOOTCONFIG_MAGIC_LEN + 8) + initrd_end - initrd_start < BOOTCONFIG_FOOTER_SIZE) return NULL; data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN; @@ -336,9 +336,64 @@ static void * __init get_boot_config_from_initrd(size_t *_size) #ifdef CONFIG_BOOT_CONFIG -/* Make an extra command line under given key word */ -static char * __init xbc_make_cmdline(const char *key) +/* Check if a parameter name matches an obs_kernel_param entry */ +static bool __init match_obs_param(const char *param, const char *pattern) { + size_t n = strlen(pattern); + + if (n > 0 && pattern[n - 1] == '=') + return parameqn(param, pattern, n - 1) && param[n - 1] == '\0'; + + return parameq(param, pattern); +} + +/* Return true if the given param is only defined by early_param(). */ +static bool __init is_early_only_param(const char *param) +{ + const struct obs_kernel_param *p; + const struct kernel_param *kp; + bool has_early = false; + + /* If handled by a regular module_param, it is not early-only */ + for (kp = __start___param; kp < __stop___param; kp++) { + if (parameq(param, kp->name)) + return false; + } + + /* Check __setup / early_param table */ + for (p = __setup_start; p < __setup_end; p++) { + if (match_obs_param(param, p->str)) { + if (!p->early) + return false; + has_early = true; + } + } + return has_early; +} + +struct xbc_filter_data { + bool warn; +}; + +static bool __init xbc_kernel_param_filter(struct xbc_node *node, + const char *key, void *data) +{ + struct xbc_filter_data *fdata = data; + + if (is_early_only_param(key)) { + if (fdata && fdata->warn) + pr_warn("bootconfig: early_param 'kernel.%s' cannot be applied from bootconfig, skipping\n", + key); + return false; + } + return true; +} + +/* Make an extra command line under given key word */ +static char * __init xbc_make_cmdline(const char *key, bool is_kernel) +{ + struct xbc_filter_data fdata = { .warn = true }; + xbc_cmdline_filter_fn filter = is_kernel ? xbc_kernel_param_filter : NULL; struct xbc_node *root; char *new_cmdline; int ret, len = 0; @@ -347,8 +402,12 @@ static char * __init xbc_make_cmdline(const char *key) if (!root) return NULL; - /* Count required buffer size */ - len = xbc_snprint_cmdline(NULL, 0, root); + /* + * Pass 1: Count required buffer size. Emit warnings for skipped early + * params in this pass so they are logged even if all parameters under + * @key are skipped and len becomes 0. + */ + len = xbc_snprint_cmdline_filter(NULL, 0, root, filter, &fdata); if (len <= 0) return NULL; @@ -358,7 +417,9 @@ static char * __init xbc_make_cmdline(const char *key) return NULL; } - ret = xbc_snprint_cmdline(new_cmdline, len + 1, root); + /* Pass 2: Render into buffer with warnings suppressed */ + fdata.warn = false; + ret = xbc_snprint_cmdline_filter(new_cmdline, len + 1, root, filter, &fdata); if (ret < 0 || ret > len) { pr_err("Failed to print extra kernel cmdline.\n"); memblock_free(new_cmdline, len + 1); @@ -432,9 +493,9 @@ static void __init setup_boot_config(void) * before this series. */ if (!from_embedded || !xbc_embedded_cmdline_applied()) - extra_command_line = xbc_make_cmdline("kernel"); + extra_command_line = xbc_make_cmdline("kernel", true); /* Also, "init." keys are init arguments */ - extra_init_args = xbc_make_cmdline("init"); + extra_init_args = xbc_make_cmdline("init", false); } return; }
diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35..98493e3 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig
@@ -17,6 +17,10 @@ config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -87,6 +91,15 @@ If you are unsure how to answer this question, answer Y. +config BPF_CRYPTO + def_bool y + depends on BPF_SYSCALL + select CRYPTO_LIB_AES_CBC + select CRYPTO_LIB_AES_ECB + help + Provide the kfuncs needed for BPF programs to encrypt and decrypt + data. The supported algorithms are AES-CBC and AES-ECB. + source "kernel/bpf/preload/Kconfig" config BPF_LSM @@ -101,4 +114,17 @@ If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem"
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d8..269810b 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile
@@ -6,7 +6,7 @@ endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o scev.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif @@ -39,6 +42,7 @@ endif ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o +obj-$(CONFIG_DEBUG_INFO_BTF_INLINE) += btf_vmlinux_inline.o endif ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o @@ -55,9 +59,7 @@ # semantics within pahole are revisited accordingly. obj-${CONFIG_BPF_LSM} += bpf_lsm_proto.o bpf_lsm.o endif -ifneq ($(CONFIG_CRYPTO),) -obj-$(CONFIG_BPF_SYSCALL) += crypto.o -endif +obj-$(CONFIG_BPF_CRYPTO) += crypto.o obj-$(CONFIG_BPF_PRELOAD) += preload/ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b68472..868589f 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c
@@ -146,7 +146,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) struct apply_range_data { struct bpf_arena *arena; - struct page **pages; + struct llist_head *pages; int i; }; @@ -158,13 +158,17 @@ struct clear_range_data { static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; + struct llist_node *node; struct page *page; pte_t pteval; if (!data) return 0; - page = d->pages[d->i]; + node = READ_ONCE(d->pages->first); + if (WARN_ON_ONCE(!node)) + return -EINVAL; + page = llist_entry(node, struct page, pcp_llist); /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; @@ -197,6 +201,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval); #endif + WARN_ON_ONCE(llist_del_first(d->pages) != node); d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; @@ -312,8 +317,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); - err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); - if (err) + arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE, true); + if (!arena->scratch_page) goto err_free_arena; range_tree_init(&arena->rt); @@ -481,7 +486,10 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; - struct page *page; + LLIST_HEAD(pages); + struct page *page, *new_page = NULL; + struct apply_range_data data; + vm_fault_t fault_ret; long kbase, kaddr; unsigned long flags; int ret; @@ -489,59 +497,123 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kbase = bpf_arena_get_kern_vm_start(arena); kaddr = kbase + (u32)(vmf->address); - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + page = vmalloc_to_page((void *)kaddr); + if (!page && !(arena->map.map_flags & BPF_F_SEGV_ON_FAULT)) { + /* + * Preallocate outside the lock so the allocation can reclaim; + * __GFP_RETRY_MAYFAIL keeps the OOM killer out of it. + */ + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + new_page = alloc_pages_node(map->numa_node, + GFP_KERNEL | __GFP_ZERO | + __GFP_ACCOUNT | __GFP_NOWARN | + __GFP_RETRY_MAYFAIL, 0); + bpf_map_memcg_exit(old_memcg, new_memcg); + } + + if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { /* * A failed lock means a possible deadlock was detected. Don't * return VM_FAULT_RETRY: this handler never took mmap_lock, but * the fault path would re-take it on retry and deadlock. Fail. */ + if (new_page) + free_pages_nolock(new_page, 0); return VM_FAULT_SIGBUS; + } page = vmalloc_to_page((void *)kaddr); if (page) { - if (page == arena->scratch_page) + if (page == arena->scratch_page) { /* BPF triggered scratch here; don't lazy-alloc over it */ - goto out_sigsegv; + fault_ret = (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) + ? VM_FAULT_SIGSEGV : VM_FAULT_SIGBUS; + goto out_err_locked; + } /* already have a page vmap-ed */ goto out; } + if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) { + /* + * User space requested to segfault when page is not allocated + * by bpf prog + */ + fault_ret = VM_FAULT_SIGSEGV; + goto out_err_locked; + } + + if (is_range_tree_set(&arena->rt, vmf->pgoff, 1)) { + /* Allocated without a page: reserved or being freed */ + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked; + } + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) - /* User space requested to segfault when page is not allocated by bpf prog */ - goto out_sigsegv_memcg; + if (!new_page) { + /* + * The probed page was freed meanwhile or preallocation failed; + * try the non-blocking allocator, we cannot sleep here. + */ + new_page = bpf_alloc_page(map->numa_node, false); + if (!new_page) { + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; + } + } ret = range_tree_clear(&arena->rt, vmf->pgoff, 1); - if (ret) - goto out_sigsegv_memcg; - - struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; - /* Account into memcg of the process that created bpf_arena */ - ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { - range_tree_set(&arena->rt, vmf->pgoff, 1); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } + llist_add(&new_page->pcp_llist, &pages); + data.arena = arena; + data.pages = &pages; + data.i = 0; ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { + llist_del_first(&pages); range_tree_set(&arena->rt, vmf->pgoff, 1); - free_pages_nolock(page, 0); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } flush_vmap_cache(kaddr, PAGE_SIZE); bpf_map_memcg_exit(old_memcg, new_memcg); + /* new_page was consumed */ + page = new_page; + new_page = NULL; out: page_ref_add(page, 1); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - vmf->page = page; - return 0; -out_sigsegv_memcg: + if (new_page) + free_pages_nolock(new_page, 0); + + /* + * A free clears the kernel pte and then takes the page lock before it + * zaps user vma-s. Either it waits for this fault and zaps its pte, or + * the page is gone from here. + */ + lock_page(page); + if (vmalloc_to_page((void *)kaddr) == page && !PageHWPoison(page)) { + vmf->page = page; + return VM_FAULT_LOCKED; + } + /* The page was freed meanwhile. Let the fault start over. */ + fault_ret = PageHWPoison(page) ? VM_FAULT_HWPOISON : VM_FAULT_NOPAGE; + unlock_page(page); + put_page(page); + return fault_ret; + +out_err_locked_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); -out_sigsegv: +out_err_locked: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - return VM_FAULT_SIGSEGV; + if (new_page) + free_pages_nolock(new_page, 0); + return fault_ret; } static const struct vm_operations_struct arena_vm_ops = { @@ -661,6 +733,27 @@ static u64 clear_lo32(u64 val) return val & ~(u64)~0U; } +static int arena_adjust_tree(struct bpf_arena *arena, long uaddr, long page_cnt, long *pgoff) +{ + int ret; + + /* Special case where user is requesting specific range. */ + if (uaddr) { + ret = is_range_tree_set(&arena->rt, *pgoff, page_cnt); + if (ret) + return ret; + return range_tree_clear(&arena->rt, *pgoff, page_cnt); + } + + ret = range_tree_find(&arena->rt, page_cnt); + if (ret < 0) + return ret; + + *pgoff = ret; + + return range_tree_clear(&arena->rt, *pgoff, page_cnt); +} + /* * Allocate pages and vmap them into kernel vmalloc area. * Later the pages will be mmaped into user space vma. @@ -673,13 +766,13 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena); struct mem_cgroup *new_memcg, *old_memcg; struct apply_range_data data; - struct page **pages = NULL; - long remaining, mapped = 0; - long alloc_pages; + LLIST_HEAD(pages); + long mapped = 0; unsigned long flags; long pgoff = 0; u32 uaddr32; - int ret, i; + long addr = 0; + int ret; if (node_id != NUMA_NO_NODE && ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) @@ -696,89 +789,63 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt /* requested address will be outside of user VMA */ return 0; } - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - /* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ - alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *)); - pages = kmalloc_nolock(alloc_pages * sizeof(struct page *), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!pages) { - bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; - } + + ret = bpf_alloc_pages(node_id, page_cnt, &pages, sleepable); + if (ret) + goto out_memcg; + data.arena = arena; - data.pages = pages; + data.pages = &pages; + data.i = 0; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; - if (uaddr) { - ret = is_range_tree_set(&arena->rt, pgoff, page_cnt); - if (ret) - goto out_unlock_free_pages; - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); - } else { - ret = pgoff = range_tree_find(&arena->rt, page_cnt); - if (pgoff >= 0) - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); + ret = arena_adjust_tree(arena, uaddr, page_cnt, &pgoff); + if (ret) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + goto out_free_pages; } - if (ret) - goto out_unlock_free_pages; - remaining = page_cnt; uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - while (remaining) { - long this_batch = min(remaining, alloc_pages); + /* + * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 + * will not overflow 32-bit. Lower 32-bit need to represent + * contiguous user address range. + * Map these pages at kern_vm_start base. + * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow + * lower 32-bit and it's ok. + */ + ret = apply_to_page_range(&init_mm, kern_vm_start + uaddr32, + page_cnt << PAGE_SHIFT, apply_range_set_cb, &data); + mapped = data.i; + if (ret) + goto out_unmap; - /* zeroing is needed, since alloc_pages_bulk() only fills in non-zero entries */ - memset(pages, 0, this_batch * sizeof(struct page *)); - - ret = bpf_map_alloc_pages(&arena->map, node_id, this_batch, pages); - if (ret) - goto out; - - /* - * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 - * will not overflow 32-bit. Lower 32-bit need to represent - * contiguous user address range. - * Map these pages at kern_vm_start base. - * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow - * lower 32-bit and it's ok. - */ - data.i = 0; - ret = apply_to_page_range(&init_mm, - kern_vm_start + uaddr32 + (mapped << PAGE_SHIFT), - this_batch << PAGE_SHIFT, apply_range_set_cb, &data); - if (ret) { - /* data.i pages were mapped, account them and free the remaining */ - mapped += data.i; - for (i = data.i; i < this_batch; i++) - free_pages_nolock(pages[i], 0); - goto out; - } - - mapped += this_batch; - remaining -= this_batch; - } flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - kfree_nolock(pages); - bpf_map_memcg_exit(old_memcg, new_memcg); - return clear_lo32(arena->user_vm_start) + uaddr32; -out: + + addr = clear_lo32(arena->user_vm_start) + uaddr32; + goto out_memcg; + +out_unmap: + /* Error handling: Undo partial mappings. */ + flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - if (mapped) { - flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); + if (mapped) arena_free_pages(arena, uaddr32, mapped, sleepable); - } - goto out_free_pages; -out_unlock_free_pages: - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + out_free_pages: - kfree_nolock(pages); + /* Error handling: Free back any unmapped pages. */ + bpf_free_pages(&pages); + +out_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; + + return addr; } /* @@ -807,7 +874,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) * otherwise call B's mark would make call A skip a vml that A has * not yet zapped for A's uaddr range. */ - mutex_lock(&arena->zap_mutex); + lockdep_assert_held(&arena->zap_mutex); mutex_lock(&arena->lock); my_gen = ++arena->zap_gen; for (;;) { @@ -841,49 +908,41 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) mutex_lock(&arena->lock); } mutex_unlock(&arena->lock); - mutex_unlock(&arena->zap_mutex); } -static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +/* + * The range stays allocated in the range tree until the pages are unmapped + * from vmalloc area and from all user space vma-s, so that nobody can + * populate it while stale TLB entries or user ptes are still around. + */ +static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt) { - struct mem_cgroup *new_memcg, *old_memcg; - u64 full_uaddr, uaddr_end; - long kaddr, pgoff; - struct page *page; - struct llist_head free_pages; - struct llist_node *pos, *t; - struct arena_free_span *s; + u64 full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + long kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; + long pgoff = compute_pgoff(arena, uaddr); struct clear_range_data cdata; + struct llist_node *pos, *t; + LLIST_HEAD(free_pages); unsigned long flags; - int ret = 0; + struct page *page; + int ret; - /* only aligned lower 32-bit are relevant */ - uaddr = (u32)uaddr; - uaddr &= PAGE_MASK; - kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; - full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; - if (full_uaddr < arena->user_vm_start) - return; - uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); - if (full_uaddr >= uaddr_end) - return; - - page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; - pgoff = compute_pgoff(arena, uaddr); - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - - if (!sleepable) - goto defer; + /* An overlapping free must not release the range before this one is done */ + guard(mutex)(&arena->zap_mutex); ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags); - - /* Can't proceed without holding the spinlock so defer the free */ if (ret) - goto defer; + return ret; - range_tree_set(&arena->rt, pgoff, page_cnt); + /* + * Take the free pages of the range too. It allocates, and can fail, + * only when the whole range is free. Then there is nothing to do. + */ + if (range_tree_clear(&arena->rt, pgoff, page_cnt)) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; + } - init_llist_head(&free_pages); cdata.arena = arena; cdata.free_pages = &free_pages; /* clear ptes and collect struct pages */ @@ -893,6 +952,12 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, /* drop the lock to do the tlb flush and zap pages */ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + /* wait for arena_vm_fault() that found the page before its pte was cleared */ + llist_for_each_entry(page, free_pages.first, pcp_llist) { + lock_page(page); + unlock_page(page); + } + /* ensure no stale TLB entries */ flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); @@ -911,8 +976,39 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, zap_pages(arena, full_uaddr, 1); __free_page(page); } - bpf_map_memcg_exit(old_memcg, new_memcg); + /* The callers hold no other rqspinlock, so this one can only time out */ + while (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + cond_resched(); + range_tree_set(&arena->rt, pgoff, page_cnt); + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; +} + +static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +{ + struct mem_cgroup *new_memcg, *old_memcg; + u64 full_uaddr, uaddr_end; + struct arena_free_span *s; + + /* only aligned lower 32-bit are relevant */ + uaddr = (u32)uaddr; + uaddr &= PAGE_MASK; + full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; + uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); + if (full_uaddr >= uaddr_end) + return; + + page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + + /* Can't proceed without holding the spinlock so defer the free */ + if (!sleepable || arena_free_range(arena, uaddr, page_cnt)) + goto defer; + + bpf_map_memcg_exit(old_memcg, new_memcg); return; defer: @@ -974,66 +1070,18 @@ static void arena_free_worker(struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); struct mem_cgroup *new_memcg, *old_memcg; - struct llist_node *list, *pos, *t; + struct llist_node *pos, *t; struct arena_free_span *s; - u64 arena_vm_start, user_vm_start; - struct llist_head free_pages; - struct clear_range_data cdata; - struct page *page; - unsigned long full_uaddr; - long kaddr, page_cnt, pgoff; - unsigned long flags; - - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { - schedule_work(work); - return; - } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - init_llist_head(&free_pages); - cdata.arena = arena; - cdata.free_pages = &free_pages; - arena_vm_start = bpf_arena_get_kern_vm_start(arena); - user_vm_start = bpf_arena_get_user_vm_start(arena); - - list = llist_del_all(&arena->free_spans); - llist_for_each(pos, list) { + llist_for_each_safe(pos, t, llist_del_all(&arena->free_spans)) { s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - kaddr = arena_vm_start + s->uaddr; - pgoff = compute_pgoff(arena, s->uaddr); - - /* clear ptes and collect pages in free_pages llist */ - apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); - } - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - - /* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */ - llist_for_each_safe(pos, t, list) { - s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - full_uaddr = clear_lo32(user_vm_start) + s->uaddr; - kaddr = arena_vm_start + s->uaddr; - - /* ensure no stale TLB entries */ - flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); - - /* remove pages from user vmas */ - zap_pages(arena, full_uaddr, page_cnt); - + while (arena_free_range(arena, s->uaddr, s->page_cnt)) + cond_resched(); kfree_nolock(s); } - /* free all pages collected by apply_to_existing_page_range() in the first loop */ - llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) { - page = llist_entry(pos, struct page, pcp_llist); - __free_page(page); - } - bpf_map_memcg_exit(old_memcg, new_memcg); }
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0ce26b5..133f189 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c
@@ -438,15 +438,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, u32 size; int cpu, off = 0; - if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) - /* unknown flags */ + if (unlikely(((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS) || + ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; if (unlikely(index >= array->map.max_entries)) /* all elements were pre-allocated, cannot insert a new one */ return -E2BIG; - if (unlikely(map_flags == BPF_NOEXIST)) + if (unlikely(map_flags & BPF_NOEXIST)) /* all elements already exist */ return -EEXIST; @@ -1035,7 +1035,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, } mutex_lock(&prog->aux->ext_mutex); - is_extended = prog->aux->is_extended; + is_extended = prog->aux->freplace_link_cnt > 0; if (!is_extended) prog->aux->prog_array_member_cnt++; mutex_unlock(&prog->aux->ext_mutex);
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 47282ff..43d5740 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c
@@ -9,7 +9,8 @@ /* for any branch, call, exit record the history of jmps in the given state */ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs) + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt) { u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; @@ -27,10 +28,13 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state env->cur_hist_ent->flags |= insn_flags; env->cur_hist_ent->spi = spi; env->cur_hist_ent->frame = frame; - verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, - "insn history: insn_idx %d linked_regs: %#llx", - env->insn_idx, env->cur_hist_ent->linked_regs); - env->cur_hist_ent->linked_regs = linked_regs; + verifier_bug_if(env->cur_hist_ent->linked_regs_cnt != 0, env, + "insn history: insn_idx %d has %u linked regs", + env->insn_idx, env->cur_hist_ent->linked_regs_cnt); + if (linked_regs_cnt) + memcpy(env->cur_hist_ent->linked_regs, linked_regs, + linked_regs_cnt * sizeof(*linked_regs)); + env->cur_hist_ent->linked_regs_cnt = linked_regs_cnt; return 0; } @@ -47,7 +51,9 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state p->flags = insn_flags; p->spi = spi; p->frame = frame; - p->linked_regs = linked_regs; + if (linked_regs_cnt) + memcpy(p->linked_regs, linked_regs, linked_regs_cnt * sizeof(*linked_regs)); + p->linked_regs_cnt = linked_regs_cnt; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; @@ -123,13 +129,26 @@ static inline void bt_reset(struct backtrack_state *bt) bt->env = env; } -static inline u32 bt_empty(struct backtrack_state *bt) +static inline bool bt_frame_stack_empty(struct backtrack_state *bt, u32 frame) { - u64 mask = 0; + return bitmap_empty(bt->stack_masks[frame], MAX_BPF_STACK_SLOTS); +} + +static inline bool bt_stack_empty(struct backtrack_state *bt) +{ + return bt_frame_stack_empty(bt, bt->frame); +} + +static inline bool bt_empty(struct backtrack_state *bt) +{ + u32 mask = 0; int i; - for (i = 0; i <= bt->frame; i++) - mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; + for (i = 0; i <= bt->frame; i++) { + mask |= bt->reg_masks[i] | bt->stack_arg_masks[i]; + if (!bt_frame_stack_empty(bt, i)) + return false; + } return mask == 0; } @@ -181,7 +200,7 @@ static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] &= ~(1ull << slot); + __clear_bit(slot, bt->stack_masks[frame]); } static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) @@ -194,14 +213,14 @@ static inline u32 bt_reg_mask(struct backtrack_state *bt) return bt->reg_masks[bt->frame]; } -static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) +static inline unsigned long *bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) { return bt->stack_masks[frame]; } -static inline u64 bt_stack_mask(struct backtrack_state *bt) +static inline unsigned long *bt_stack_mask(struct backtrack_state *bt) { - return bt->stack_masks[bt->frame]; + return bt_frame_stack_mask(bt, bt->frame); } static inline u8 bt_stack_arg_mask(struct backtrack_state *bt) @@ -233,17 +252,16 @@ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) break; } } -/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) + +/* format stack slots bitmask, e.g., "-8,-24,-40" for slots 0, 2 and 4 */ +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask) { - DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; - bitmap_from_u64(mask, stack_mask); - for_each_set_bit(i, mask, 64) { + for_each_set_bit(i, stack_mask, MAX_BPF_STACK_SLOTS) { n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); first = false; buf += n; @@ -406,15 +424,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { - if (bpf_pseudo_call(insn)) { - int subprog_insn_idx, subprog; + if (bpf_pseudo_call(insn) || bpf_is_callx(insn)) { + int subprog_insn_idx, subprog = -1; - subprog_insn_idx = idx + insn->imm + 1; - subprog = bpf_find_subprog(env, subprog_insn_idx); - if (subprog < 0) - return -EFAULT; + if (bpf_pseudo_call(insn)) { + subprog_insn_idx = idx + insn->imm + 1; + subprog = bpf_find_subprog(env, subprog_insn_idx); + if (subprog < 0) + return -EFAULT; + } - if (bpf_subprog_is_global(env, subprog)) { + /* callx calls static subprogs only */ + if (subprog >= 0 && bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog @@ -423,6 +444,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +457,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -450,10 +473,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, /* we are now tracking register spills correctly, * so any instance of leftover slots is a bug */ - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, - "static subprog leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "static subprog leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* propagate r1-r5 to the caller */ @@ -486,9 +510,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, "callback leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "callback leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* clear r1-r5 in callback subprog's mask */ @@ -506,6 +532,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,40 +548,51 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool from_subprog_call, r0_precise; + bool from_subprog_call, r0_precise, r2_precise; /* BPF_EXIT in subprog or callback always returns * right after the call instruction, so by checking * whether the instruction at subseq_idx-1 is subprog * call or not we can distinguish actual exit from * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. */ from_subprog_call = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + (bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) || + bpf_is_callx(&env->prog->insnsi[subseq_idx - 1])); r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. - * In case of a regular function call, instructions giving - * precision to registers R1-R5 should have been found already. + * In case of a regular function call, the callee defines the + * return registers R0 and R2, so clear them before checking + * that instructions giving precision to registers R1-R5 have + * been found already. R2 is also an argument register, hence + * it has to be cleared before that check. * In case of a callback from bpf_loop(), R{1,4} in the calling - * frame would be set as precise and that is correct. + * frame would be set as precise and that is correct, and R2 + * might be a precise helper argument as well, so leave it set. */ + bt_clear_reg(bt, BPF_REG_0); + if (from_subprog_call) + bt_clear_reg(bt, BPF_REG_2); if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ @@ -693,10 +732,12 @@ void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, i, j); } } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE || reg->precise) continue; reg->precise = true; @@ -808,6 +849,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, int subseq_idx = -1; struct bpf_func_state *func; bool tmp, skip_first = true; + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int i, fr, err; @@ -846,6 +888,10 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } if (last_idx < 0) { + /* Consecutive checkpoints can have no instructions between them. */ + if (st->parent) + goto parent; + /* we are at the entry into subprog, which * is expected for global funcs, but only if * requested precise registers are R1-R5 @@ -854,7 +900,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, if (st->curframe == 0 && st->frame[0]->subprogno > 0 && st->frame[0]->callsite == BPF_MAIN_FUNC && - bt_stack_mask(bt) == 0 && + bt_stack_empty(bt) && (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { bitmap_from_u64(mask, bt_reg_mask(bt)); for_each_set_bit(i, mask, 32) { @@ -868,8 +914,9 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return 0; } - verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", - st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); + verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %s", + st->frame[0]->subprogno, bt_reg_mask(bt), env->tmp_str_buf); return -EFAULT; } @@ -909,6 +956,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return -EFAULT; } } +parent: st = st->parent; if (!st) break; @@ -930,18 +978,18 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } } - bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); - for_each_set_bit(i, mask, 64) { - if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, + for_each_set_bit(i, bt_frame_stack_mask(bt, fr), MAX_BPF_STACK_SLOTS) { + if (verifier_bug_if(i >= bpf_stack_nr_slots(func), env, "stack slot %d, total slots %d", - i, func->allocated_stack / BPF_REG_SIZE)) + i, bpf_stack_nr_slots(func))) return -EFAULT; - if (!bpf_is_spilled_scalar_reg(&func->stack[i])) { + ss = bpf_stack_slot(func, i); + if (!bpf_is_spilled_scalar_reg(ss)) { bt_clear_frame_slot(bt, fr, i); continue; } - reg = &func->stack[i].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->precise) { bt_clear_frame_slot(bt, fr, i); } else {
diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index a2f84afe..92bbb71 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c
@@ -199,12 +199,16 @@ int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog) return -EBUSY; /* - * Reset all the map indexes to the original values. This is needed, - * e.g., when a replay of verification with different log level should - * be performed. + * Reset the map to its pre-verification state. The xlated and jitted + * offsets and the jitted target pointers are recomputed by the verifier + * and the JIT for this program, so any values left by a previous owner + * must be cleared here. */ - for (i = 0; i < map->max_entries; i++) + for (i = 0; i < map->max_entries; i++) { values[i].xlated_off = values[i].orig_off; + values[i].jitted_off = 0; + insn_array->ips[i] = 0; + } return 0; } @@ -228,10 +232,11 @@ void bpf_insn_array_release(struct bpf_map *map) { struct bpf_insn_array *insn_array = cast_insn_array(map); - atomic_set(&insn_array->used, 0); + /* Paired with atomic_xchg() in bpf_insn_array_init(). */ + atomic_set_release(&insn_array->used, 0); } -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len) { struct bpf_insn_array *insn_array = cast_insn_array(map); int i; @@ -240,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) return; for (i = 0; i < map->max_entries; i++) { - if (insn_array->values[i].xlated_off <= off) + if (insn_array->values[i].xlated_off < first) continue; if (insn_array->values[i].xlated_off == INSN_DELETED) continue;
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index d9191df..f0b29a1 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c
@@ -72,6 +72,15 @@ static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_ return tinfo->reg_info->feature & BPF_ITER_RESCHED; } +/* Only allow sleepable program for resched-able iterator that can run one */ +static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo, + const struct bpf_prog *prog) +{ + return !prog->sleepable || + (bpf_iter_target_support_resched(tinfo) && + !(tinfo->reg_info->feature & BPF_ITER_NOSLEEP)); +} + static bool bpf_iter_support_resched(struct seq_file *seq) { struct bpf_iter_priv_data *iter_priv; @@ -342,6 +351,13 @@ int bpf_iter_prog_supported(struct bpf_prog *prog) const char *prefix = BPF_ITER_FUNC_PREFIX; int prefix_len = strlen(prefix); + /* + * The attach function of a program that has a target program is a + * function of that program, whatever its name is. + */ + if (prog->aux->dst_prog) + return -EINVAL; + if (strncmp(attach_fname, prefix, prefix_len)) return -EINVAL; @@ -409,6 +425,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, struct bpf_prog *new_prog, struct bpf_prog *old_prog) { + struct bpf_iter_link *iter_link = + container_of(link, struct bpf_iter_link, link); int ret = 0; mutex_lock(&link_mutex); @@ -419,11 +437,19 @@ static int bpf_iter_link_replace(struct bpf_link *link, if (link->prog->type != new_prog->type || link->prog->expected_attach_type != new_prog->expected_attach_type || - link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { + link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id || + !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) { ret = -EINVAL; goto out_unlock; } + if (iter_link->tinfo->reg_info->check_prog) { + ret = iter_link->tinfo->reg_info->check_prog(new_prog, + &iter_link->aux); + if (ret) + goto out_unlock; + } + old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); @@ -544,8 +570,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, if (!tinfo) return -ENOENT; - /* Only allow sleepable program for resched-able iterator */ - if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) + if (!bpf_iter_target_support_prog(tinfo, prog)) return -EINVAL; link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN);
diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b..b12fcec 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c
@@ -199,7 +199,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * } if (pin_owner) - refcount_dec(&local_storage->owner_refcnt); + bpf_local_storage_unpin_owner(local_storage); } /* local_storage->lock must be held and selem->local_storage == local_storage. @@ -240,24 +240,26 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, hlist_add_head_rcu(&selem->snode, &local_storage->list); } +/* Must be called with the owning local_storage->lock held. */ static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) { struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; struct bpf_local_storage_map_bucket *b; - unsigned long flags; int err; + lockdep_assert_irqs_disabled(); + local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held()); smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, flags); + err = raw_res_spin_lock(&b->lock); if (err) return err; hlist_del_init_rcu(&selem->map_node); - raw_res_spin_unlock_irqrestore(&b->lock, flags); + raw_res_spin_unlock(&b->lock); return 0; } @@ -350,7 +352,8 @@ int bpf_selem_unlink(struct bpf_local_storage_elem *selem) static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, struct bpf_local_storage_map_bucket *b) { - bool in_map_free = !!b, free_storage = false; + bool in_map_free = !!b, free_storage = false, sleepable_users = false; + bool keep_fields = false; struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; unsigned long flags; @@ -360,17 +363,26 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); if (smap) { + /* A map that is freed, or NULL here, has no program left. */ + sleepable_users = !in_map_free && + (atomic64_read(&smap->map.sleepable_refcnt) || + READ_ONCE(smap->map.was_inner_map)); b = b ? : select_bucket(smap, local_storage); err = raw_res_spin_lock_irqsave(&b->lock, flags); if (!err) { /* * Call bpf_obj_free_fields() under b->lock to make sure it is done * exactly once for an selem. Safe to free special fields immediately - * as no BPF program should be referencing the selem. + * as no BPF program should be referencing the selem, unless it + * is sleepable: see below. */ if (likely(selem_linked_to_map(selem))) { hlist_del_init_rcu(&selem->map_node); - bpf_obj_free_fields(smap->map.record, SDATA(selem)->data); + if (sleepable_users) + keep_fields = true; + else + bpf_obj_free_fields(smap->map.record, + SDATA(selem)->data); unlink++; } raw_res_spin_unlock_irqrestore(&b->lock, flags); @@ -385,7 +397,15 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * with selem1 as raw_res_spin_lock_irqsave() returns -ETIMEDOUT. */ WARN_ON_ONCE(err && in_map_free); - if (!err || in_map_free) + /* + * A sleepable program can still use, and fill, the fields of a value + * that it got before it let go of the owner. As for bpf_selem_unlink(), + * bpf_selem_free_trace_rcu() frees them, and takes smap for that: + * map_free() waits for this RCU read side critical section and then + * for the callback. It must not take smap away from an selem that + * destroy() has unlinked from the map. + */ + if (in_map_free ? err || unlink : !err && !keep_fields) RCU_INIT_POINTER(SDATA(selem)->smap, NULL); } @@ -429,13 +449,18 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * then freed after an RCU grace period. However, if destroy() and map_free() are * racing or rqspinlock returns errors in unlikely situations (unlink != 2), free * the selem only after both map_free() and destroy() see the selem. + * + * The owner that map_free() takes an selem from is alive: sleepable code, which + * is in no RCU read side critical section, may be walking its list for another + * map. And it can still use a value that it got before it let go of an owner + * that destroy() is called for. */ if (unlink == 2 || atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED) - bpf_selem_free(selem, true); + bpf_selem_free(selem, !in_map_free && !sleepable_users); if (free_storage) - bpf_local_storage_free(local_storage, true); + bpf_local_storage_free(local_storage, !in_map_free); } void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, @@ -507,8 +532,6 @@ int bpf_local_storage_alloc(void *owner, if (err) goto uncharge; - bpf_selem_link_map_nolock(b, first_selem); - owner_storage_ptr = (struct bpf_local_storage **)owner_storage(smap, owner); /* Publish storage to the owner. @@ -523,11 +546,17 @@ int bpf_local_storage_alloc(void *owner, */ prev_storage = cmpxchg(owner_storage_ptr, NULL, storage); if (unlikely(prev_storage)) { - bpf_selem_unlink_map_nolock(first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); err = -EAGAIN; goto uncharge; } + /* + * Not before the storage is published: the owner does not wait for + * users of a storage that it knows nothing of, so that nobody may find + * such a storage through the map. b->lock holds off those who find + * first_selem through the owner and want to unlink it. + */ + bpf_selem_link_map_nolock(b, first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); return 0; @@ -552,7 +581,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, struct bpf_local_storage *local_storage; struct bpf_local_storage_map_bucket *b; HLIST_HEAD(old_selem_free_list); - unsigned long flags, b_flags; + unsigned long flags; int err; /* BPF_EXIST and BPF_NOEXIST cannot be both set */ @@ -637,7 +666,8 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, b_flags); + /* local_storage->lock is held, so IRQs are already disabled. */ + err = raw_res_spin_lock(&b->lock); if (err) goto unlock; @@ -655,7 +685,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, &old_selem_free_list); } - raw_res_spin_unlock_irqrestore(&b->lock, b_flags); + raw_res_spin_unlock(&b->lock); unlock: raw_res_spin_unlock_irqrestore(&local_storage->lock, flags); free_selem: @@ -750,8 +780,8 @@ u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage) while (refcount_read(&local_storage->owner_refcnt)) cpu_relax(); /* - * Paired with refcount_dec() in bpf_selem_unlink_nofail() - * to make sure destroy() sees the correct local_storage->mem_charge. + * Paired with bpf_local_storage_unpin_owner() to make sure + * destroy() sees the correct local_storage->mem_charge. */ smp_mb(); }
diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index 5ed7cb4..3858886 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c
@@ -71,7 +71,7 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l, return; bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); bpf_lru_node_clear_ref(node); /* Reset pending_free only when moving to the free list */ if (tgt_type == BPF_LRU_LIST_T_FREE) @@ -94,7 +94,7 @@ static void __bpf_lru_node_move(struct bpf_lru_list *l, if (node->type != tgt_type) { bpf_lru_list_count_dec(l, node->type); bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); } bpf_lru_node_clear_ref(node); @@ -298,9 +298,6 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l, { unsigned long flags; - if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) - return; - if (raw_res_spin_lock_irqsave(&l->lock, flags)) { WRITE_ONCE(node->pending_free, 1); return; @@ -542,7 +539,7 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru, return; } - if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { + if (unlikely(READ_ONCE(node->type) != BPF_LRU_LOCAL_LIST_T_PENDING)) { raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); goto check_lru_list;
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 82c59884..f58dce8 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c
@@ -116,8 +116,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, } #endif +BTF_ID_LIST_SINGLE(bpf_lsm_inode_init_security_btf_id, func, + bpf_lsm_inode_init_security) + int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { u32 btf_id = prog->aux->attach_btf_id; const char *func_name = prog->aux->attach_func_name; @@ -140,6 +143,23 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, return -EINVAL; } + if (btf_id == bpf_lsm_inode_init_security_btf_id[0]) { + /* + * inode, dir, qstr, xattrs, xattr_count + * + * The trusted pointer this hands the program is only as + * trustworthy as the context it is loaded from, which + * lsm_verifier_ops keeps read-only. + */ + static const struct bpf_ctx_arg_aux xattr_count_arg_info = { + .offset = 4 * sizeof(u64), + .reg_type = PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED, + .mem_size = sizeof(int), + }; + + return bpf_prog_ctx_arg_info_init(prog, &xattr_count_arg_info, 1); + } + return 0; } @@ -414,8 +434,8 @@ const struct bpf_prog_ops lsm_prog_ops = { }; const struct bpf_verifier_ops lsm_verifier_ops = { - .get_func_proto = bpf_lsm_func_proto, - .is_valid_access = btf_ctx_access, + .get_func_proto = bpf_lsm_func_proto, + .is_valid_access = bpf_tracing_btf_ctx_access, }; /* hooks return 0 or 1 */
diff --git a/kernel/bpf/bpf_lsm_proto.c b/kernel/bpf/bpf_lsm_proto.c index 44a54fd..b81179a 100644 --- a/kernel/bpf/bpf_lsm_proto.c +++ b/kernel/bpf/bpf_lsm_proto.c
@@ -4,6 +4,7 @@ */ #include <linux/fs.h> +#include <linux/xattr.h> #include <linux/bpf_lsm.h> /* @@ -12,8 +13,22 @@ * explicitly enforces that BPF LSM programs check for NULL before attempting to * dereference it. */ -int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, - unsigned long prot, unsigned long flags) +noinline int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, + unsigned long prot, unsigned long flags) { return 0; } + +/* + * Strong definition of the inode_init_security() BPF LSM hook. Both the + * qstr and the xattr array are NULL for some callers, so the __nullable + * suffix marks it as PTR_MAYBE_NULL. BPF LSM programs have to check before + * dereferencing them or handing them to bpf_inode_init_xattr(). + */ +noinline int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr__nullable, + struct xattr *xattrs__nullable, + int *xattr_count) +{ + return -EOPNOTSUPP; +}
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index d7c3030..1178acd 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c
@@ -13,6 +13,7 @@ #include <linux/btf_ids.h> #include <linux/rcupdate_wait.h> #include <linux/poll.h> +#include <linux/bpf-cgroup.h> struct bpf_struct_ops_value { struct bpf_struct_ops_common_value common; @@ -57,7 +58,7 @@ struct bpf_struct_ops_map { struct bpf_struct_ops_link { struct bpf_link link; - struct bpf_map __rcu *map; + struct bpf_map *map; wait_queue_head_t wait_hup; }; @@ -1024,9 +1025,18 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_map_area_free(st_map); } +static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + + bpf_struct_ops_map_del_ksyms(st_map); +} + static void bpf_struct_ops_map_free(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + struct bpf_struct_ops *st_ops = st_map->st_ops_desc->st_ops; + bool tasks_rcu = st_ops->free_after_tasks_rcu_gp; /* st_ops->owner was acquired during map_alloc to implicitly holds * the btf's refcnt. The acquire was only done when btf_is_module() @@ -1037,24 +1047,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) bpf_struct_ops_map_dissoc_progs(st_map); - bpf_struct_ops_map_del_ksyms(st_map); - - /* The struct_ops's function may switch to another struct_ops. - * - * For example, bpf_tcp_cc_x->init() may switch to - * another tcp_cc_y by calling - * setsockopt(TCP_CONGESTION, "tcp_cc_y"). - * During the switch, bpf_struct_ops_put(tcp_cc_x) is called - * and its refcount may reach 0 which then free its - * trampoline image while tcp_cc_x is still running. - * - * A vanilla rcu gp is to wait for all bpf-tcp-cc prog - * to finish. bpf-tcp-cc prog is non sleepable. - * A rcu_tasks gp is to wait for the last few insn - * in the tramopline image to finish before releasing - * the trampoline image. - */ - synchronize_rcu_mult(call_rcu, call_rcu_tasks); + if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) + synchronize_rcu_tasks(); __bpf_struct_ops_map_free(map); } @@ -1123,6 +1117,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) goto errout; } + if (st_ops_desc->st_ops->cgroup_atype && !(attr->map_flags & BPF_F_LINK)) { + ret = -EOPNOTSUPP; + goto errout; + } + vt = st_ops_desc->value_type; if (attr->value_size != vt->size) { ret = -EINVAL; @@ -1163,6 +1162,8 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); + map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; + map->free_after_rcu_gp = true; return map; @@ -1195,6 +1196,7 @@ const struct bpf_map_ops bpf_struct_ops_map_ops = { .map_alloc_check = bpf_struct_ops_map_alloc_check, .map_alloc = bpf_struct_ops_map_alloc, .map_free = bpf_struct_ops_map_free, + .map_free_pre_rcu = bpf_struct_ops_map_free_pre_rcu, .map_get_next_key = bpf_struct_ops_map_get_next_key, .map_lookup_elem = bpf_struct_ops_map_lookup_elem, .map_delete_elem = bpf_struct_ops_map_delete_elem, @@ -1281,7 +1283,31 @@ int bpf_struct_ops_for_each_prog(const void *kdata, } EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog); -static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->kvalue.data; +} + +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cgroup_atype; +} + +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cfi_stubs; +} + +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; @@ -1297,8 +1323,7 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) struct bpf_struct_ops_map *st_map; st_link = container_of(link, struct bpf_struct_ops_link, link); - st_map = (struct bpf_struct_ops_map *) - rcu_dereference_protected(st_link->map, true); + st_map = (struct bpf_struct_ops_map *)st_link->map; if (st_map) { st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); bpf_map_put(&st_map->map); @@ -1313,11 +1338,11 @@ static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) seq_printf(seq, "map_id:\t%d\n", map->id); - rcu_read_unlock(); + mutex_unlock(&update_mutex); } static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, @@ -1327,11 +1352,31 @@ static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) info->struct_ops.map_id = map->id; - rcu_read_unlock(); + mutex_unlock(&update_mutex); + return 0; +} + +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + struct bpf_struct_ops_map *st_map, *old_st_map; + + if (!old_map) + return -ENOLINK; + if (expected_old_map && old_map != expected_old_map) + return -EPERM; + + st_map = container_of(new_map, struct bpf_struct_ops_map, map); + old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); + /* The new and old struct_ops must be the same type. */ + if (st_map->st_ops_desc != old_st_map->st_ops_desc) + return -EINVAL; + return 0; } @@ -1353,30 +1398,19 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map return -EOPNOTSUPP; mutex_lock(&update_mutex); - - old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); - if (!old_map) { - err = -ENOLINK; + old_map = st_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) goto err_out; - } - if (expected_old_map && old_map != expected_old_map) { - err = -EPERM; - goto err_out; - } old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); - /* The new and old struct_ops must be the same type. */ - if (st_map->st_ops_desc != old_st_map->st_ops_desc) { - err = -EINVAL; - goto err_out; - } err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link); if (err) goto err_out; bpf_map_inc(new_map); - rcu_assign_pointer(st_link->map, new_map); + WRITE_ONCE(st_link->map, new_map); bpf_map_put(old_map); err_out: @@ -1393,7 +1427,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) mutex_lock(&update_mutex); - map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); + map = st_link->map; if (!map) { mutex_unlock(&update_mutex); return 0; @@ -1402,7 +1436,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); - RCU_INIT_POINTER(st_link->map, NULL); + WRITE_ONCE(st_link->map, NULL); /* Pair with bpf_map_get() in bpf_struct_ops_link_create() or * bpf_map_inc() in bpf_struct_ops_map_link_update(). */ @@ -1422,7 +1456,7 @@ static __poll_t bpf_struct_ops_map_link_poll(struct file *file, poll_wait(file, &st_link->wait_hup, pts); - return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP; + return READ_ONCE(st_link->map) ? 0 : EPOLLHUP; } static const struct bpf_link_ops bpf_struct_ops_map_lops = { @@ -1440,6 +1474,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) struct bpf_link_primer link_primer; struct bpf_struct_ops_map *st_map; struct bpf_map *map; + int cgroup_atype; int err; map = bpf_map_get(attr->link_create.map_fd); @@ -1453,6 +1488,19 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) goto err_out; } + cgroup_atype = st_map->st_ops_desc->st_ops->cgroup_atype; + if (cgroup_atype) { + err = cgroup_bpf_struct_ops_attach(map, attr); + bpf_map_put(map); + return err; + } + + if (memchr_inv(&attr->link_create.cgroup, 0, sizeof(attr->link_create.cgroup)) || + attr->link_create.target_fd) { + err = -EINVAL; + goto err_out; + } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; @@ -1478,7 +1526,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) link = NULL; goto err_out; } - RCU_INIT_POINTER(link->map, map); + link->map = map; mutex_unlock(&update_mutex); return bpf_link_settle(&link_primer);
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d870bc5..538129e 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c
@@ -20,15 +20,17 @@ #include <linux/btf.h> #include <linux/btf_ids.h> #include <linux/bpf.h> +#include <linux/bpf-cgroup.h> #include <linux/bpf_lsm.h> #include <linux/skmsg.h> #include <linux/perf_event.h> #include <linux/bsearch.h> -#include <linux/kobject.h> #include <linux/string.h> -#include <linux/sysfs.h> #include <linux/overflow.h> #include <linux/bitops.h> +#include <linux/unaligned.h> +#include <linux/vmalloc.h> +#include <crypto/sha2.h> #include <net/netfilter/nf_bpf_link.h> @@ -345,6 +347,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; const char *btf_type_str(const struct btf_type *t) @@ -517,11 +522,27 @@ static bool btf_type_is_decl_tag(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; } +static bool btf_type_is_loc_param(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PARAM; +} + +static bool btf_type_is_loc_proto(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PROTO; +} + +static bool btf_type_is_locsec(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOCSEC; +} + static bool btf_type_nosize(const struct btf_type *t) { return btf_type_is_void(t) || btf_type_is_fwd(t) || btf_type_is_func(t) || btf_type_is_func_proto(t) || - btf_type_is_decl_tag(t); + btf_type_is_decl_tag(t) || btf_type_is_loc_param(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t); } static bool btf_type_nosize_or_null(const struct btf_type *t) @@ -768,7 +789,9 @@ static bool btf_type_is_resolve_source_only(const struct btf_type *t) { return btf_type_is_var(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* What types need to be resolved? @@ -796,7 +819,9 @@ static bool btf_type_needs_resolve(const struct btf_type *t) btf_type_is_var(t) || btf_type_is_func(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* t->size can be used */ @@ -877,16 +902,6 @@ static bool btf_name_offset_valid(const struct btf *btf, u32 offset) return offset < btf->hdr.str_len; } -static bool __btf_name_char_ok(char c, bool first) -{ - if ((first ? !isalpha(c) : - !isalnum(c)) && - c != '_' && - c != '.') - return false; - return true; -} - const char *btf_str_by_offset(const struct btf *btf, u32 offset) { while (offset < btf->start_str_off) @@ -899,20 +914,21 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset) return NULL; } +/* Names in BTF of Rust are not C identifiers. Allow any printable character */ static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) { /* offset must be valid */ const char *src = btf_str_by_offset(btf, offset); const char *src_limit; - if (!__btf_name_char_ok(*src, true)) + if (!isprint(*src)) return false; /* set a limit on identifier length */ src_limit = src + KSYM_NAME_LEN; src++; while (*src && src < src_limit) { - if (!__btf_name_char_ok(*src, false)) + if (!isprint(*src)) return false; src++; } @@ -1065,6 +1081,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, } #define BTF_SHOW_MAX_ITER 10 +/* nested structs and arrays shown; __btf_show_indent() has as many columns */ +#define BTF_SHOW_MAX_DEPTH 32 #define BTF_KIND_BIT(kind) (1ULL << kind) @@ -1458,8 +1476,18 @@ static void *btf_show_start_aggr_type(struct btf_show *show, const struct btf_type *t, u32 type_id, void *data) { - void *safe_data = btf_show_start_type(show, t, type_id, data); + void *safe_data; + /* + * Structs and arrays are shown by recursion. BTF loading bounds the + * depth of one resolve walk, not how deep resolved types may nest. + */ + if (show->state.depth >= BTF_SHOW_MAX_DEPTH) { + show->state.status = -E2BIG; + return NULL; + } + + safe_data = btf_show_start_type(show, t, type_id, data); if (!safe_data) return safe_data; @@ -1920,7 +1948,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) { btf->base_btf = (struct btf *)base_btf; btf->start_id = btf_nr_types(base_btf); - btf->start_str_off = base_btf->hdr.str_len; + btf->start_str_off = base_btf->start_str_off + base_btf->hdr.str_len; } static int env_resolve_init(struct btf_verifier_env *env) @@ -2064,7 +2092,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_id: id of u32 * *total_nelems: (x * y). Hence, individual elem size is * (*type_size / *total_nelems) - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. * * type: is not an array (e.g. const struct X) * return type: type "struct X" @@ -2072,7 +2101,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_type: same as return type ("struct X") * *elem_id: 0 * *total_nelems: 1 - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. */ static const struct btf_type * __btf_resolve_size(const struct btf *btf, const struct btf_type *type, @@ -2105,7 +2135,8 @@ __btf_resolve_size(const struct btf *btf, const struct btf_type *type, case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_TYPE_TAG: - id = type->type; + if (!array_type) + id = type->type; type = btf_type_by_id(btf, type->type); break; @@ -3538,6 +3569,28 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +/* Whether the pointee of pointer type @t carries the type tag @name */ +static bool btf_ptr_has_type_tag(const struct btf *btf, const struct btf_type *t, const char *name) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), name)) + return true; + } + + return false; +} + +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + return btf_ptr_has_type_tag(btf, t, "arena"); +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -3679,6 +3732,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_TIMER, "bpf_timer", true }, { BPF_WORKQUEUE, "bpf_wq", true }, { BPF_TASK_WORK, "bpf_task_work", true }, + { BPF_RCU_HEAD, "bpf_rcu_head", true }, { BPF_LIST_HEAD, "bpf_list_head", false }, { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, @@ -3864,6 +3918,7 @@ static int btf_find_field_one(const struct btf *btf, case BPF_RB_NODE: case BPF_REFCOUNT: case BPF_TASK_WORK: + case BPF_RCU_HEAD: ret = btf_find_struct(btf, var_type, off, sz, field_type, info_cnt ? &info[0] : &tmp); if (ret < 0) @@ -4159,6 +4214,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->wq_off = -EINVAL; rec->refcount_off = -EINVAL; rec->task_work_off = -EINVAL; + rec->rcu_head_off = -EINVAL; for (i = 0; i < cnt; i++) { field_type_size = btf_field_type_size(info_arr[i].type); if (info_arr[i].off + field_type_size > value_size) { @@ -4202,6 +4258,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type WARN_ON_ONCE(rec->task_work_off >= 0); rec->task_work_off = rec->fields[i].offset; break; + case BPF_RCU_HEAD: + WARN_ON_ONCE(rec->rcu_head_off >= 0); + rec->rcu_head_off = rec->fields[i].offset; + break; case BPF_REFCOUNT: WARN_ON_ONCE(rec->refcount_off >= 0); /* Cache offset for faster lookup at runtime */ @@ -4747,6 +4807,336 @@ static const struct btf_kind_operations enum64_ops = { .show = btf_enum64_show, }; +static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 size, meta_needed, vlen = btf_vlen(t); + + meta_needed = sizeof(*p) + sizeof(__u32) * vlen; + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + size = t->size; + if (!size || size > 16) { + btf_verifier_log_type(env, t, "Unexpected size"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + /* All LOC_PARAMs have vlen of at least 1 */ + if (vlen < 1) + goto invalid_vlen; + + switch (p->flags) { + case BTF_LOC_PARAM_CONST: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED: + if (vlen < 1 || vlen > 2) + goto invalid_vlen; + if (!size || size > sizeof(__u64)) + goto invalid_vlen; + /* Ensure size/vlen are consistent */ + if (vlen != (size > sizeof(__u32) ? 2 : 1)) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG: + /* Reg or reg/reg pair */ + if (vlen > 2) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF: + /* A reg deref without offset can only have vlen 1 */ + if (vlen != 1) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED: + if (vlen < 2 || vlen > 3) + goto invalid_vlen; + break; + default: + btf_verifier_log_type(env, t, "Invalid flags"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; + +invalid_vlen: + btf_verifier_log_type(env, t, "Invalid vlen"); + return -EINVAL; +} + +static void btf_loc_param_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 i, vlen = btf_vlen(t); + + btf_verifier_log(env, "size=%u vlen=%u flags=0x%x", t->size, vlen, p->flags); + for (i = 0; i < vlen; i++) + btf_verifier_log(env, ", %u", p->values[i]); +} + +static const struct btf_kind_operations loc_param_ops = { + .check_meta = btf_loc_param_check_meta, + .resolve = btf_df_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_param_log, + .show = btf_df_show, +}; + +static s32 btf_loc_proto_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 meta_needed; + + meta_needed = sizeof(__u32) * btf_type_vlen(t); + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; +} + +static void btf_loc_proto_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + + btf_verifier_log(env, "vlen=%u", nr_params); + for (i = 0; i < nr_params; i++, params++) + btf_verifier_log(env, ", %u", *params); +} + +static int btf_loc_proto_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + struct btf *btf = env->btf; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid loc_proto type"); + return -EINVAL; + } + + for (i = 0; i < nr_params; i++) { + const struct btf_type *param_type; + u32 param_type_id = params[i]; + + if (!param_type_id) + continue; + + param_type = btf_type_by_id(btf, param_type_id); + if (!param_type || !btf_type_is_loc_param(param_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_param#%u", i + 1); + return -EINVAL; + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations loc_proto_ops = { + .check_meta = btf_loc_proto_check_meta, + .resolve = btf_loc_proto_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_proto_log, + .show = btf_df_show, +}; + +__printf(4, 5) +static void btf_verifier_log_loc(struct btf_verifier_env *env, + const struct btf_type *locsec_type, + const struct btf_loc *loc, + const char *fmt, ...) +{ + struct bpf_verifier_log *log = &env->log; + va_list args; + + if (!bpf_verifier_log_needed(log)) + return; + if (log->level == BPF_LOG_KERNEL && !fmt) + return; + if (env->phase != CHECK_META) + btf_verifier_log_type(env, locsec_type, NULL); + + __btf_verifier_log(log, "\t func=%u loc_proto=%u offset=%u", + loc->func, loc->loc_proto, loc->offset); + if (fmt && *fmt) { + __btf_verifier_log(log, " "); + va_start(args, fmt); + bpf_verifier_vlog(log, fmt, args); + va_end(args); + } + + __btf_verifier_log(log, "\n"); +} + +static s32 btf_locsec_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 i, meta_needed, vlen = btf_type_vlen(t); + const struct btf_loc *loc; + + meta_needed = sizeof(struct btf_loc) * vlen; + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + if (!t->name_off || + !btf_name_valid_section(env->btf, t->name_off)) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + for_each_loc(i, t, loc) { + /* A loc func, loc proto cannot be in type void */ + if (!loc->func || !BTF_TYPE_ID_VALID(loc->func)) { + btf_verifier_log_loc(env, t, loc, "Invalid func"); + return -EINVAL; + } + if (!loc->loc_proto || !BTF_TYPE_ID_VALID(loc->loc_proto)) { + btf_verifier_log_loc(env, t, loc, "Invalid loc_proto"); + return -EINVAL; + } + btf_verifier_log_loc(env, t, loc, NULL); + } + + return meta_needed; +} + +static void btf_locsec_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + btf_verifier_log(env, "vlen=%u", btf_type_vlen(t)); +} + +static int btf_locsec_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const struct btf_loc *loc; + struct btf *btf = env->btf; + u32 i; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid locsec type"); + return -EINVAL; + } + + env->resolve_mode = RESOLVE_TBD; + for (i = v->next_member, loc = btf_type_loc_secinfo(t) + i; + i < btf_type_vlen(t); i++, loc++) { + const struct btf_type *func_type, *func_proto_type; + const struct btf_type *loc_proto_type; + u32 func_type_id = loc->func; + u32 loc_proto_type_id = loc->loc_proto; + u32 proto_vlen; + + func_type = btf_type_by_id(btf, func_type_id); + if (!func_type || !btf_type_is_func(func_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + func_proto_type = btf_type_by_id(btf, func_type->type); + if (!func_proto_type || !btf_type_is_func_proto(func_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + proto_vlen = btf_vlen(func_proto_type); + + if (!env_type_is_resolved(env, func_type_id)) { + env_stack_set_next_member(env, i); + return env_stack_push(env, func_type, func_type_id); + } + + loc_proto_type = btf_type_by_id(btf, loc_proto_type_id); + if (!loc_proto_type || !btf_type_is_loc_proto(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_proto#%u", i + 1); + return -EINVAL; + } + if (proto_vlen != btf_vlen(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Mismatched vlen for loc_proto#%u", i + 1); + return -EINVAL; + } + + if (!env_type_is_resolved(env, loc_proto_type_id)) { + env_stack_set_next_member(env, i + 1); + return env_stack_push(env, loc_proto_type, + loc_proto_type_id); + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations locsec_ops = { + .check_meta = btf_locsec_check_meta, + .resolve = btf_locsec_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_locsec_log, + .show = btf_df_show, +}; + static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, const struct btf_type *t, u32 meta_left) @@ -5029,7 +5419,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, env->resolve_mode = RESOLVE_TBD; for_each_vsi_from(i, v->next_member, v->t, vsi) { - u32 var_type_id = vsi->type, type_id, type_size = 0; + u32 var_type_id = vsi->type, type_id; const struct btf_type *var_type = btf_type_by_id(env->btf, var_type_id); if (!var_type || !btf_type_is_var(var_type)) { @@ -5044,16 +5434,16 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, return env_stack_push(env, var_type, var_type_id); } + /* + * The variable can be smaller than its type. It's a piece of + * a variable that the compiler split then, with the type of + * the whole variable. + */ type_id = var_type->type; - if (!btf_type_id_size(btf, &type_id, &type_size)) { + if (!btf_type_id_size(btf, &type_id, NULL)) { btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); return -EINVAL; } - - if (vsi->size < type_size) { - btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); - return -EINVAL; - } } env_stack_pop_resolved(env, 0, 0); @@ -5066,6 +5456,16 @@ static void btf_datasec_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } +/* A piece of a variable is smaller than its type, which is the one of the whole variable */ +static bool btf_var_is_piece(const struct btf *btf, const struct btf_type *var, + const struct btf_var_secinfo *vsi) +{ + u32 size; + + return IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, var->type), &size)) || + vsi->size < size; +} + static void btf_datasec_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, @@ -5073,6 +5473,7 @@ static void btf_datasec_show(const struct btf *btf, { const struct btf_var_secinfo *vsi; const struct btf_type *var; + bool comma = false; u32 i; if (!btf_show_start_type(show, t, type_id, data)) @@ -5082,8 +5483,12 @@ static void btf_datasec_show(const struct btf *btf, __btf_name_by_offset(btf, t->name_off)); for_each_vsi(i, t, vsi) { var = btf_type_by_id(btf, vsi->type); - if (i) + /* there is less data than the type takes */ + if (btf_var_is_piece(btf, var, vsi)) + continue; + if (comma) btf_show(show, ","); + comma = true; btf_type_ops(var)->show(btf, var, vsi->type, data + vsi->offset, bits_offset, show); } @@ -5384,6 +5789,10 @@ static int btf_func_check(struct btf_verifier_env *env, return -EINVAL; } + /* Rust leaves out names of some arguments of static functions */ + if (btf_func_linkage(t) == BTF_FUNC_STATIC) + return 0; + args = (const struct btf_param *)(proto_type + 1); nr_args = btf_type_vlen(proto_type); for (i = 0; i < nr_args; i++) { @@ -5416,6 +5825,9 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = &decl_tag_ops, [BTF_KIND_TYPE_TAG] = &modifier_ops, [BTF_KIND_ENUM64] = &enum64_ops, + [BTF_KIND_LOC_PARAM] = &loc_param_ops, + [BTF_KIND_LOC_PROTO] = &loc_proto_ops, + [BTF_KIND_LOCSEC] = &locsec_ops, }; static s32 btf_check_meta(struct btf_verifier_env *env, @@ -5490,7 +5902,8 @@ static bool btf_resolve_valid(struct btf_verifier_env *env, if (!env_type_is_resolved(env, type_id)) return false; - if (btf_type_is_struct(t) || btf_type_is_datasec(t)) + if (btf_type_is_struct(t) || btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t)) return !btf_resolved_type_id(btf, type_id) && !btf_resolved_type_size(btf, type_id); @@ -5815,7 +6228,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env) return btf_check_sec_info(env, btf_data_size); } -static const char *alloc_obj_fields[] = { +static const char * const alloc_obj_fields[] = { "bpf_spin_lock", "bpf_list_head", "bpf_list_node", @@ -6530,51 +6943,22 @@ __u32 btf_relocate_id(const struct btf *btf, __u32 id) #ifdef CONFIG_DEBUG_INFO_BTF_MODULES -static struct btf *btf_parse_module(const char *module_name, const void *data, - unsigned int data_size, void *base_data, - unsigned int base_data_size) +static struct btf *btf_parse_split(struct btf_verifier_env *env, + const char *name, const void *data, + unsigned int data_size, struct btf *base_btf) { - struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL; - struct btf_verifier_env *env = NULL; - struct bpf_verifier_log *log; - int err = 0; - - vmlinux_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(vmlinux_btf)) - return vmlinux_btf; - if (!vmlinux_btf) - return ERR_PTR(-EINVAL); - - env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); - if (!env) - return ERR_PTR(-ENOMEM); - - log = &env->log; - log->level = BPF_LOG_KERNEL; - - if (base_data) { - base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); - if (IS_ERR(base_btf)) { - err = PTR_ERR(base_btf); - goto errout; - } - } else { - base_btf = vmlinux_btf; - } + struct btf *btf; + int err; btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); - if (!btf) { - err = -ENOMEM; - goto errout; - } + if (!btf) + return ERR_PTR(-ENOMEM); env->btf = btf; - btf->base_btf = base_btf; - btf->start_id = base_btf->nr_types; - btf->start_str_off = base_btf->hdr.str_len; + btf_set_base_btf(btf, base_btf); btf->kernel_btf = true; btf->named_start_id = 0; - strscpy(btf->name, module_name); + strscpy(btf->name, name); btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); if (!btf->data) { @@ -6601,28 +6985,180 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, if (err) goto errout; + return btf; + +errout: + btf_free(btf); + return ERR_PTR(err); +} + +static int btf_rebase_inline(struct btf *inline_btf, + const struct btf *module_btf, + const u32 *module_id_map, + const u32 *module_str_map, + u32 old_module_type_cnt) +{ + u32 old_start_id = inline_btf->start_id; + u32 old_start_str_off = inline_btf->start_str_off; + u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len; + u32 new_start_id = btf_nr_types(module_btf); + u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len; + s64 id_delta = (s64)new_start_id - old_start_id; + s64 str_delta = (s64)new_start_str_off - old_start_str_off; + u32 i; + + /* + * The inline BTF was parsed relative to the original module BTF. Its + * base IDs must therefore use the map generated when that BTF was + * relocated, while IDs for inline-local types only move by the change + * in the module BTF's starting ID. + */ + for (i = 0; i < inline_btf->nr_types; i++) { + struct btf_field_iter it; + struct btf_type *t = inline_btf->types[i]; + u32 *id, *str_off; + int err; + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); + if (err) + return err; + while ((id = btf_field_iter_next(&it))) { + if (!*id) + continue; + if (*id < old_module_type_cnt) { + if (module_id_map) + *id = module_id_map[*id]; + } else if (*id >= old_start_id) { + *id += id_delta; + } else { + return -EINVAL; + } + } + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); + if (err) + return err; + while ((str_off = btf_field_iter_next(&it))) { + if (!*str_off) + continue; + if (*str_off < old_module_start_str_off) { + /* Vmlinux strings retain their offsets for in-tree modules. */ + if (!module_id_map) + continue; + if (!module_str_map || !module_str_map[*str_off]) + return -EINVAL; + *str_off = module_str_map[*str_off]; + continue; + } + *str_off += str_delta; + } + } + + btf_set_base_btf(inline_btf, module_btf); + btf_check_sorted(inline_btf); + return 0; +} + +static struct btf *btf_parse_module(const char *module_name, const void *data, + unsigned int data_size, void *base_data, + unsigned int base_data_size, + const void *inline_data, + unsigned int inline_data_size, + void **relocated_inline_data) +{ + struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL; + struct btf_verifier_env *env = NULL; + struct bpf_verifier_log *log; + u32 *module_str_map = NULL; + u32 old_module_type_cnt; + int err = 0; + + *relocated_inline_data = NULL; + vmlinux_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(vmlinux_btf)) + return vmlinux_btf; + if (!vmlinux_btf) + return ERR_PTR(-EINVAL); + + env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); + if (!env) + return ERR_PTR(-ENOMEM); + + log = &env->log; + log->level = BPF_LOG_KERNEL; + + if (base_data) { + base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); + if (IS_ERR(base_btf)) { + err = PTR_ERR(base_btf); + goto errout; + } + } else { + base_btf = vmlinux_btf; + } + + btf = btf_parse_split(env, module_name, data, data_size, base_btf); + if (IS_ERR(btf)) { + err = PTR_ERR(btf); + btf = NULL; + goto errout; + } + old_module_type_cnt = btf_nr_types(btf); + + if (inline_data_size) { + inline_btf = btf_parse_split(env, module_name, inline_data, + inline_data_size, btf); + if (IS_ERR(inline_btf)) { + pr_warn("failed to validate module [%s] inline BTF: %ld\n", + module_name, PTR_ERR(inline_btf)); + inline_btf = NULL; + } + } + if (base_btf != vmlinux_btf) { - err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); + err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, + &module_str_map); if (err) goto errout; btf_free(base_btf); base_btf = vmlinux_btf; } - btf_verifier_env_free(env); + if (inline_btf) { + err = btf_rebase_inline(inline_btf, btf, btf->base_id_map, + module_str_map, old_module_type_cnt); + if (err) { + pr_warn("failed to relocate module [%s] inline BTF: %d\n", + module_name, err); + btf_free(inline_btf); + } else { + *relocated_inline_data = inline_btf->data; + inline_btf->data = NULL; + btf_free(inline_btf); + } + } + + /* + * With a distilled base, btf_relocate() replaces the base BTF and + * rewrites string offsets. Check ordering only after that final BTF + * view has been established, so named_start_id describes the BTF used + * by name lookups. + */ btf_check_sorted(btf); + btf_verifier_env_free(env); + kvfree(module_str_map); refcount_set(&btf->refcnt, 1); return btf; errout: + kvfree(module_str_map); btf_verifier_env_free(env); + if (inline_btf) + btf_free(inline_btf); if (!IS_ERR(base_btf) && base_btf != vmlinux_btf) btf_free(base_btf); - if (btf) { - kvfree(btf->data); - kvfree(btf->types); - kfree(btf); - } + if (btf) + btf_free(btf); return ERR_PTR(err); } @@ -7016,8 +7552,9 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } /* - * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or - * PTR_TO_ARENA (both nullable and non-nullable cases). + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL, + * PTR_TO_ARENA (both nullable and non-nullable cases) or fixed-size + * PTR_TO_MEM. */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; @@ -7027,8 +7564,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, flag = type_flag(ctx_arg_info->reg_type); if (ctx_arg_info->offset == off && (type == PTR_TO_ARENA || + type == PTR_TO_MEM || (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; + info->mem_size = ctx_arg_info->mem_size; return true; } } @@ -7155,18 +7694,60 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } EXPORT_SYMBOL_GPL(btf_ctx_access); +#define MAX_ARRAYS_WALK 16 + +/* Description of an array crossed while walking a BTF access chain */ +struct array_access { + u32 id; /* BTF_KIND_ARRAY type id */ + u32 off; /* offset of the access within the array */ +}; + enum bpf_struct_walk_result { /* < 0 error */ WALK_SCALAR = 0, WALK_PTR, WALK_PTR_UNTRUSTED, + WALK_PTR_ARENA, WALK_STRUCT, }; +static const struct btf_member *find_flex_member(const struct btf *btf, const struct btf_type *t) +{ + const struct btf_array *array; + const struct btf_type *mtype; + const struct btf_member *member; + u32 vlen = btf_type_vlen(t); + + if (vlen == 0) + return NULL; + + member = btf_type_member(t) + vlen - 1; + mtype = btf_type_skip_modifiers(btf, member->type, NULL); + if (!btf_type_is_array(mtype)) + return NULL; + + array = (const struct btf_array *)(mtype + 1); + if (array->nelems != 0) + return NULL; + + return member; +} + +static int push_array(struct array_access *arrays, u32 *arrays_cnt, struct array_access item) +{ + if (arrays == NULL) + return 0; + if (*arrays_cnt == MAX_ARRAYS_WALK) + return -1; + arrays[(*arrays_cnt)++] = item; + return 0; +} + static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name, bool walk_flex_arrays) + const char **field_name, bool walk_flex_arrays, + struct array_access *arrays, u32 *arrays_cnt) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7201,23 +7782,20 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, /* If the last element is a variable size array, we may * need to relax the rule. */ - if (vlen == 0) - goto error; - - member = btf_type_member(t) + vlen - 1; - mtype = btf_type_skip_modifiers(btf, member->type, - NULL); - if (!btf_type_is_array(mtype)) - goto error; - - array_elem = (struct btf_array *)(mtype + 1); - if (array_elem->nelems != 0) + member = find_flex_member(btf, t); + if (!member) goto error; moff = __btf_member_bit_offset(t, member) / 8; if (off < moff) goto error; + mtype = btf_type_skip_modifiers(btf, member->type, &mid); + array_elem = (struct btf_array *)(mtype + 1); + + if (push_array(arrays, arrays_cnt, (struct array_access){ mid, off - moff })) + return -E2BIG; + /* allow structure and integer */ t = btf_type_skip_modifiers(btf, array_elem->type, NULL); @@ -7339,6 +7917,9 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, * the array's element as long as it is * within the mtrue_end boundary. */ + if (push_array(arrays, arrays_cnt, + (struct array_access){ mid, off - moff })) + return -E2BIG; /* skip empty array */ if (moff == mtrue_end) @@ -7399,6 +7980,38 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, id = ctx.id; stype = ctx.t; + /* + * __nullable combines with any of the above: it only says + * that the pointer may be NULL. The caller decides whether + * that matters for the register type it derives. + */ + if (btf_ptr_has_type_tag(btf, mtype, "nullable")) + tmp_flag |= PTR_MAYBE_NULL; + + /* + * An __arena member of a kernel struct holds the kernel + * address of arena memory, which the verifier turns into + * an arena pointer on load. Program BTF is not affected: + * there the tag is what the compiler emits for the arena + * address space. An offset into a union resolves to its + * first member, so a program reading another member there + * would get an arena pointer it did not ask for; the walk + * marked such a union untrusted. + */ + if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { + if (*flag & PTR_UNTRUSTED) { + bpf_log(log, + "arena pointer member %s of %s is inside a union\n", + mname, tname); + return -EACCES; + } + *next_btf_id = id; + *flag |= tmp_flag & PTR_MAYBE_NULL; + if (field_name) + *field_name = mname; + return WALK_PTR_ARENA; + } + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -7431,15 +8044,29 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, int btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, - int off, int size, enum bpf_access_type atype __maybe_unused, + int _off, int size, enum bpf_access_type atype __maybe_unused, u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name) { const struct btf *btf = reg->btf; enum bpf_type_flag tmp_flag = 0; + struct array_access arrays[MAX_ARRAYS_WALK]; + const struct btf_member *member; const struct btf_type *t; + u32 i, arrays_cnt = 0; u32 id = reg->btf_id; - int err; + s64 min_off, max_off, flex_off; + int err, off, ret; + + if (check_add_overflow(reg_smin(reg), (s64)_off, &min_off) || + check_add_overflow(reg_smax(reg), (s64)_off, &max_off) || + min_off != (int)min_off) { + bpf_log(log, + "offset computation overflows: register offset range is [%lld, %lld], instruction offset is %d\n", + reg_smin(reg), reg_smax(reg), _off); + return -EINVAL; + } + off = min_off; while (type_is_alloc(reg->type)) { struct btf_struct_meta *meta; @@ -7466,26 +8093,35 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, - field_name, !type_is_alloc(reg->type)); - + field_name, !type_is_alloc(reg->type), arrays, &arrays_cnt); switch (err) { case WALK_PTR: /* For local types, the destination register cannot * become a pointer again. */ - if (type_is_alloc(reg->type)) - return SCALAR_VALUE; + if (type_is_alloc(reg->type)) { + ret = SCALAR_VALUE; + goto check_variable_offset; + } /* If we found the pointer or scalar on t+off, * we're done. */ *next_btf_id = id; *flag = tmp_flag; - return PTR_TO_BTF_ID; + ret = PTR_TO_BTF_ID; + goto check_variable_offset; case WALK_PTR_UNTRUSTED: *flag = MEM_RDONLY | PTR_UNTRUSTED; - return PTR_TO_MEM; + ret = PTR_TO_MEM; + goto check_variable_offset; + case WALK_PTR_ARENA: + *next_btf_id = id; + *flag = tmp_flag; + ret = PTR_TO_ARENA; + goto check_variable_offset; case WALK_SCALAR: - return SCALAR_VALUE; + ret = SCALAR_VALUE; + goto check_variable_offset; case WALK_STRUCT: /* We found nested struct, so continue the search * by diving in it. At this point the offset is @@ -7505,6 +8141,55 @@ int btf_struct_access(struct bpf_verifier_log *log, } while (t); return -EINVAL; + +check_variable_offset: + if (min_off == max_off) + return ret; + + /* Find an offset at which access would go to a flexible array tail (if any). */ + t = btf_type_skip_modifiers(btf, reg->btf_id, NULL); + member = find_flex_member(btf, t); + flex_off = member ? __btf_member_bit_offset(t, member) / 8 : S64_MAX; + + /* + * If this is a varying offset access, the step recorded within a register + * should correspond to one of the arrays visited while walking. + */ + for (i = 0; i < arrays_cnt; i++) { + s64 array_start, array_end, access_end; + u32 asize, esize, elem_id; + + /* + * 'arrays' records top-level arrays only, use __btf_resolve_size() + * to get flattened representation. + */ + t = btf_type_by_id(btf, arrays[i].id); + if (IS_ERR(__btf_resolve_size(btf, t, &asize, NULL, &elem_id, NULL, NULL)) || + IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, elem_id), &esize))) + continue; + /* Make sure every accessed offset lands on the same field of some element. */ + if (esize == 0 || reg->step % esize != 0) + continue; + + array_start = min_off - arrays[i].off; + /* An array within the flexible tail has no upper bound to exceed */ + if (array_start >= flex_off && btf_type_is_array(t) && + btf_type_array(t)->nelems == 0) + return ret; + + /* the 'size' bytes read at max_off must stay within the array */ + if (check_add_overflow(array_start, (s64)asize, &array_end) || + check_add_overflow(max_off, (s64)size, &access_end) || + access_end > array_end) + continue; + return ret; + } + + bpf_log(log, + "invalid variable offset access into struct %s: offsets [%lld, %lld] size %d step %d do not match any array\n", + btf_name_by_offset(btf, btf_type_by_id(btf, reg->btf_id)->name_off), + min_off, max_off, size, reg->step); + return -EINVAL; } /* Check that two BTF types, each specified as an BTF object + id, are exactly @@ -7545,7 +8230,7 @@ bool btf_struct_ids_match(struct bpf_verifier_log *log, if (!type) return false; err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, - walk_flex_arrays); + walk_flex_arrays, NULL, NULL); if (err != WALK_STRUCT) return false; @@ -7601,6 +8286,9 @@ static u8 __get_arg_fmodel_flags(const struct btf *btf, { u8 flags = __get_type_fmodel_flags(t); + if (btf_func_arg_align(btf, t) > sizeof(u64)) + flags |= BTF_FMODEL_ALIGN16_ARG; + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; else if (btf_param_match_suffix(btf, arg, "__arena")) @@ -7644,7 +8332,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7737,6 +8425,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); @@ -7981,51 +8675,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -8033,16 +8695,62 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. + */ + bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; + + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) + return 0; + } + return -EOPNOTSUPP; } +static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname, + bool is_global, u32 slot_cnt, + struct bpf_subprog_info *sub) +{ + if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + if (is_global) { + bpf_log(log, + "global function %s() needs %d > %d argument slots, " + "stack args not supported\n", + tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS); + return -EINVAL; + } + if (!bpf_jit_supports_stack_args()) { + bpf_log(log, "JIT does not support function %s() with %d argument slots\n", + tname, slot_cnt); + return -EFAULT; + } + sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS; + return 0; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -8062,7 +8770,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) const struct btf_param *args; const struct btf_type *t, *ref_t, *fn_t; int err; - u32 i, nargs, btf_id; + u32 i, slots_used, nargs, btf_id; const char *tname; if (sub->args_cached) @@ -8106,34 +8814,29 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) } args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); - sub->arg_cnt = nargs; + sub->arg_slot_cnt = nargs; if (nargs > MAX_BPF_FUNC_ARGS) { bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", MAX_BPF_FUNC_ARGS, tname, nargs); return -EFAULT; } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - if (!bpf_jit_supports_stack_args()) { - bpf_log(log, "JIT does not support function %s() with %d args\n", - tname, nargs); - return -EFAULT; - } - sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; - } + err = btf_check_arg_slots(log, tname, is_global, nargs, sub); + if (err) + return err; - if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { - bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", - tname, nargs, MAX_BPF_FUNC_REG_ARGS); - return -EINVAL; - } - - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", - tname); + "Global function %s() has unsupported return type. " + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } @@ -8141,15 +8844,17 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) /* Convert BTF function arguments into verifier types. * Only PTR_TO_CTX and SCALAR are supported atm. */ - for (i = 0; i < nargs; i++) { + for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + + if (slots_used >= MAX_BPF_FUNC_ARGS) + goto too_many_slots; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) @@ -8166,7 +8871,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) btf_validate_prog_ctx_type(log, btf, t, i, prog_type, prog->expected_attach_type)) return -EINVAL; - sub->args[i].arg_type = ARG_PTR_TO_CTX; + sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX; continue; } if (btf_is_dynptr_ptr(btf, t)) { @@ -8174,7 +8879,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_DYNPTR; + sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR; continue; } if (tags & ARG_TAG_TRUSTED) { @@ -8189,10 +8894,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (kern_type_id < 0) return kern_type_id; - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; if (tags & ARG_TAG_NULLABLE) - sub->args[i].arg_type |= PTR_MAYBE_NULL; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type |= PTR_MAYBE_NULL; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_UNTRUSTED) { @@ -8206,8 +8912,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) ref_t = btf_type_skip_modifiers(btf, t->type, NULL); if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) { - sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; - sub->args[i].mem_size = 0; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | + PTR_UNTRUSTED; + sub->args[slots_used].mem_size = 0; + slots_used++; continue; } @@ -8223,8 +8931,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) i, btf_type_str(ref_t), tname); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_ARENA) { @@ -8232,7 +8941,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_ARENA; + sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA; continue; } if (is_global) { /* generic user data pointer */ @@ -8252,10 +8961,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; if (tags & ARG_TAG_NONNULL) - sub->args[i].arg_type &= ~PTR_MAYBE_NULL; - sub->args[i].mem_size = mem_size; + sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL; + sub->args[slots_used].mem_size = mem_size; + slots_used++; continue; } @@ -8264,8 +8974,32 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i); return -EINVAL; } - if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[i].arg_type = ARG_ANYTHING; + if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) { + u32 nslots; + + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + if (!is_global) + return -EINVAL; + bpf_log(log, + "Arg#%d type %s in %s() has size %u, only 1 to %d bytes " + "can be passed by value\n", + i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (btf_type_is_struct(t) && + !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (!is_global) + return -EINVAL; + bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", + i, btf_type_str(t), tname); + return -EINVAL; + } + + nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + if (slots_used + nslots > MAX_BPF_FUNC_ARGS) + goto too_many_slots; + while (nslots--) + sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } if (!is_global) @@ -8275,9 +9009,21 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } + err = btf_check_arg_slots(log, tname, is_global, slots_used, sub); + if (err) + return err; + sub->arg_slot_cnt = slots_used; + sub->args_cached = true; return 0; + +too_many_slots: + if (!is_global) + return -EINVAL; + bpf_log(log, "Arguments of %s() need more than %d argument slots\n", + tname, MAX_BPF_FUNC_ARGS); + return -EINVAL; } static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, @@ -8552,6 +9298,8 @@ struct btf_module { struct module *module; struct btf *btf; struct bin_attribute *sysfs_attr; + struct bin_attribute *sysfs_inline_attr; + void *btf_inline_data; int flags; }; @@ -8560,28 +9308,149 @@ static DEFINE_MUTEX(btf_module_mutex); static void purge_cand_cache(struct btf *btf); +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline_link[]; +extern char __stop_BTF_inline_link[]; + +static struct bin_attribute *vmlinux_inline_attr; +static const struct btf_link *vmlinux_inline_link; + +static const struct btf_link *btf_parse_link(void *start, unsigned int size) +{ + const struct btf_link *link = start; + unsigned int module_name_len; + + if (!link || size != sizeof(*link) || link->btf_size == 0) + return NULL; + + module_name_len = strnlen(link->module_name, sizeof(link->module_name)); + if (!module_name_len || module_name_len == sizeof(link->module_name) || + memchr_inv(link->module_name + module_name_len + 1, 0, + sizeof(link->module_name) - module_name_len - 1)) + return NULL; + return link; +} + +static bool btf_verify_link(const struct btf_link *link, const struct module *mod, + const void *data, size_t data_size) +{ + u8 actual_digest[BTF_LINK_SHA256_LEN]; + + if (!link || data_size != link->btf_size) { + pr_warn(" BTF link from module [%s] missing or has unexpected size\n", mod->name); + return false; + } + sha256(data, data_size, actual_digest); + if (memcmp(actual_digest, link->sha256, sizeof(actual_digest))) { + pr_warn("BTF link from module [%s] has unexpected digest\n", mod->name); + return false; + } + + return true; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ + vmlinux_inline_link = btf_parse_link(__start_BTF_inline_link, + __stop_BTF_inline_link - + __start_BTF_inline_link); + if (!vmlinux_inline_link) { + pr_warn("invalid vmlinux inline BTF link\n"); + return; + } + vmlinux_inline_attr = sysfs_btf_add("vmlinux.inline", NULL, + vmlinux_inline_link->btf_size, true, + vmlinux_inline_link->module_name); + if (IS_ERR(vmlinux_inline_attr)) { + pr_warn("failed to register vmlinux inline BTF in sysfs: %ld\n", + PTR_ERR(vmlinux_inline_attr)); + vmlinux_inline_attr = NULL; + } +} + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return vmlinux_inline_link && !strcmp(mod->name, vmlinux_inline_link->module_name); +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + void *data; + + if (!mod->btf_inline_data || + mod->btf_inline_data_size != vmlinux_inline_link->btf_size) + return -EINVAL; + + data = vmalloc_user(mod->btf_inline_data_size); + if (!data) + return -ENOMEM; + memcpy(data, mod->btf_inline_data, mod->btf_inline_data_size); + + if (!btf_verify_link(vmlinux_inline_link, mod, data, mod->btf_inline_data_size) || + !sysfs_btf_update(vmlinux_inline_attr, data)) { + kvfree(data); + return -EINVAL; + } + return 0; +} + +#else + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return false; +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + return 0; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ +} + +#endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) */ + static int btf_module_notify(struct notifier_block *nb, unsigned long op, void *module) { struct btf_module *btf_mod, *tmp; struct module *mod = module; + struct bin_attribute *attr; struct btf *btf; + void *inline_data = NULL, *relocated_inline_data = NULL; + unsigned int inline_data_size = 0; int err = 0; - if (mod->btf_data_size == 0 || - (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && - op != MODULE_STATE_GOING)) + if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && + op != MODULE_STATE_GOING) goto out; switch (op) { case MODULE_STATE_COMING: + if (btf_module_is_vmlinux_inline(mod)) { + err = btf_module_handle_vmlinux_inline(mod); + break; + } + +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + if (mod->btf_inline_data_size) { + inline_data = mod->btf_inline_data; + inline_data_size = mod->btf_inline_data_size; + } +#endif + if (!mod->btf_data_size) + break; btf_mod = kzalloc_obj(*btf_mod); if (!btf_mod) { err = -ENOMEM; goto out; } btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, - mod->btf_base_data, mod->btf_base_data_size); + mod->btf_base_data, mod->btf_base_data_size, + inline_data, inline_data_size, + &relocated_inline_data); if (IS_ERR(btf)) { kfree(btf_mod); if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) { @@ -8596,6 +9465,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, err = btf_alloc_id(btf); if (err) { btf_free(btf); + kvfree(relocated_inline_data); kfree(btf_mod); goto out; } @@ -8604,37 +9474,35 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_lock(&btf_module_mutex); btf_mod->module = module; btf_mod->btf = btf; + btf_mod->btf_inline_data = relocated_inline_data; list_add(&btf_mod->list, &btf_modules); mutex_unlock(&btf_module_mutex); - if (IS_ENABLED(CONFIG_SYSFS)) { - struct bin_attribute *attr; - - attr = kzalloc_obj(*attr); - if (!attr) - goto out; - - sysfs_bin_attr_init(attr); - attr->attr.name = btf->name; - attr->attr.mode = 0444; - attr->size = btf->data_size; - attr->private = btf->data; - attr->read = sysfs_bin_attr_simple_read; - - err = sysfs_create_bin_file(btf_kobj, attr); - if (err) { - pr_warn("failed to register module [%s] BTF in sysfs: %d\n", - mod->name, err); - kfree(attr); - err = 0; - goto out; - } - - btf_mod->sysfs_attr = attr; + attr = sysfs_btf_add(btf->name, btf->data, btf->data_size, false, NULL); + if (IS_ERR(attr)) { + err = 0; + goto out; } + btf_mod->sysfs_attr = attr; + if (relocated_inline_data) { + char name[MODULE_NAME_LEN + sizeof(".inline")]; + + snprintf(name, sizeof(name), "%s.inline", mod->name); + attr = sysfs_btf_add(name, relocated_inline_data, + inline_data_size, false, NULL); + if (IS_ERR(attr)) { + kvfree(relocated_inline_data); + btf_mod->btf_inline_data = NULL; + err = 0; + break; + } + btf_mod->sysfs_inline_attr = attr; + } break; case MODULE_STATE_LIVE: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -8646,6 +9514,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_unlock(&btf_module_mutex); break; case MODULE_STATE_GOING: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -8660,10 +9530,12 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, btf_free_id(btf_mod->btf); list_del(&btf_mod->list); if (btf_mod->sysfs_attr) - sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); + sysfs_btf_remove(btf_mod->sysfs_attr); + if (btf_mod->sysfs_inline_attr) + sysfs_btf_remove(btf_mod->sysfs_inline_attr); + kvfree(btf_mod->btf_inline_data); purge_cand_cache(btf_mod->btf); btf_put(btf_mod->btf); - kfree(btf_mod->sysfs_attr); kfree(btf_mod); break; } @@ -8680,6 +9552,7 @@ static struct notifier_block btf_module_nb = { static int __init btf_module_init(void) { + btf_vmlinux_inline_lazy_init(); register_module_notifier(&btf_module_nb); return 0; } @@ -9953,6 +10826,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, struct bpf_verifier_log *log) { struct btf_struct_ops_tab *tab, *new_tab; + int cgroup_atype; int i, err; tab = btf->struct_ops_tab; @@ -9964,8 +10838,10 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, btf->struct_ops_tab = tab; } + cgroup_atype = st_ops->cgroup_atype; for (i = 0; i < tab->cnt; i++) - if (tab->ops[i].st_ops == st_ops) + if (tab->ops[i].st_ops == st_ops || + (cgroup_atype && cgroup_atype == tab->ops[i].st_ops->cgroup_atype)) return -EEXIST; if (tab->cnt == tab->capacity) { @@ -9985,6 +10861,31 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, if (err) return err; + if (cgroup_atype) { + /* + * Cgroup struct_ops callers hold the RCU read lock around the + * entire trampoline call, including its trailing instructions. + * A regular RCU grace period therefore protects both the kdata + * and the trampoline image, so a tasks RCU grace period is not + * needed. + */ + if (!cgroup_bpf_is_struct_ops_atype(cgroup_atype) || + st_ops->reg || st_ops->unreg || st_ops->free_after_tasks_rcu_gp) { + bpf_struct_ops_desc_release(&tab->ops[btf->struct_ops_tab->cnt]); + return -EINVAL; + } + + /* + * There is no need to unregister from the cgroup when btf_free() + * runs. No struct_ops map or cgroup link can be created once its + * BTF is gone. + */ + cgroup_bpf_struct_ops_register(cgroup_atype, + tab->ops[btf->struct_ops_tab->cnt].type_id, + st_ops->cfi_stubs, + st_ops->free_after_mult_rcu_gp); + } + btf->struct_ops_tab->cnt++; return 0;
diff --git a/kernel/bpf/btf_vmlinux_inline.c b/kernel/bpf/btf_vmlinux_inline.c new file mode 100644 index 0000000..d5c599d --- /dev/null +++ b/kernel/bpf/btf_vmlinux_inline.c
@@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ +/* + * Provide kernel BTF inline function information for use by BPF tools. + */ +#include <linux/btf.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline[]; +extern char __stop_BTF_inline[]; +#endif + +static int __init btf_vmlinux_inline_init(void) +{ +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) + size_t data_size = __stop_BTF_inline - __start_BTF_inline; + + if (data_size) + sysfs_btf_add("vmlinux.inline", __start_BTF_inline, data_size, true, NULL); +#endif + return 0; +} +subsys_initcall(btf_vmlinux_inline_init); + +MODULE_DESCRIPTION("BTF inline information for vmlinux"); +MODULE_LICENSE("GPL");
diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 842c7d1..2ee950e 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c
@@ -9,6 +9,8 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +#define BPF_MAX_GOTOX_EDGES BPF_COMPLEXITY_LIMIT_INSNS + /* non-recursive DFS pseudo code * 1 procedure DFS-iterative(G,v): * 2 label v as discovered @@ -284,15 +286,17 @@ static struct bpf_iarray *jt_from_map(struct bpf_map *map) } /* - * Find and collect all maps which fit in the subprog. Return the result as one - * combined jump table in jt->items (allocated with kvcalloc) + * Collect the jump table of every subprogram that has one, as the combined + * table of all maps whose targets land inside that subprogram. All gotox + * instructions of a subprogram share the same table, so this is done in a + * single pass over the maps rather than once per gotox. */ -static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, - int subprog_start, int subprog_end) +static int __compute_subprog_jts(struct bpf_verifier_env *env) { - struct bpf_iarray *jt = NULL; + struct bpf_subprog_info *subprog; + struct bpf_iarray *jt, *jt_cur; struct bpf_map *map; - struct bpf_iarray *jt_cur; + u32 old_cnt; int i; for (i = 0; i < env->insn_array_map_cnt; i++) { @@ -303,73 +307,100 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, map = env->insn_array_maps[i]; jt_cur = jt_from_map(map); - if (IS_ERR(jt_cur)) { - kvfree(jt); - return jt_cur; + if (IS_ERR(jt_cur)) + return PTR_ERR(jt_cur); + + subprog = bpf_find_containing_subprog(env, jt_cur->items[0]); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %u of an insn array map", + jt_cur->items[0])) { + kvfree(jt_cur); + return -EFAULT; + } + if (jt_cur->items[jt_cur->cnt - 1] >= (subprog + 1)->start) { + verbose(env, "jump table of subprog starting at %u spans multiple subprogs, map id=%d\n", + subprog->start, map->id); + bpf_diag_program_structure( + env, subprog->start, "jump table spans subprograms", + "Keep every entry of a jump table inside one subprogram.", + "The jump table in map id %d for the subprogram that starts at instruction %u reaches past its end at instruction %u.", + map->id, subprog->start, (subprog + 1)->start); + kvfree(jt_cur); + return -EINVAL; } - /* - * This is enough to check one element. The full table is - * checked to fit inside the subprog later in create_jt() - */ - if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) { - u32 old_cnt = jt ? jt->cnt : 0; - jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt); - if (!jt) { - kvfree(jt_cur); - return ERR_PTR(-ENOMEM); - } - memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + old_cnt = subprog->jt ? subprog->jt->cnt : 0; + jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt); + if (!jt) { + subprog->jt = NULL; + kvfree(jt_cur); + return -ENOMEM; } + memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + subprog->jt = jt; kvfree(jt_cur); } - if (!jt) { + for (i = 0; i < env->subprog_cnt; i++) { + jt = env->subprog_info[i].jt; + if (jt) + jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); + } + + env->cfg.subprog_jts_ready = true; + return 0; +} + +/* Leave nothing behind on failure. */ +static int compute_subprog_jts(struct bpf_verifier_env *env) +{ + int err; + + err = __compute_subprog_jts(env); + if (err) + bpf_free_subprog_jts(env); + return err; +} + +void bpf_free_subprog_jts(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(env->subprog_info); i++) { + kvfree(env->subprog_info[i].jt); + env->subprog_info[i].jt = NULL; + } + env->cfg.subprog_jts_ready = false; +} + +static struct bpf_iarray * +subprog_jt(int t, struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog; + int subprog_start, err; + + if (!env->cfg.subprog_jts_ready) { + err = compute_subprog_jts(env); + if (err) + return ERR_PTR(err); + } + + subprog = bpf_find_containing_subprog(env, t); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", t)) + return ERR_PTR(-EFAULT); + subprog_start = subprog->start; + + if (!subprog->jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); bpf_diag_program_structure( - env, subprog_start, "missing jump table", + env, t, "missing jump table", "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", "No jump table was found for the subprogram that starts at instruction %u.", subprog_start); return ERR_PTR(-EINVAL); } - jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); - return jt; -} - -static struct bpf_iarray * -create_jt(int t, struct bpf_verifier_env *env) -{ - struct bpf_subprog_info *subprog; - int subprog_start, subprog_end; - struct bpf_iarray *jt; - int i; - - subprog = bpf_find_containing_subprog(env, t); - subprog_start = subprog->start; - subprog_end = (subprog + 1)->start; - jt = jt_from_subprog(env, subprog_start, subprog_end); - if (IS_ERR(jt)) - return jt; - - /* Check that the every element of the jump table fits within the given subprogram */ - for (i = 0; i < jt->cnt; i++) { - if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { - verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", - t, subprog_start, subprog_end); - bpf_diag_program_structure( - env, t, "jump table target out of range", - "Keep every jump-table target inside the same subprogram.", - "The jump table for instruction %d points outside subprogram range [%u,%u).", - t, subprog_start, subprog_end); - kvfree(jt); - return ERR_PTR(-EINVAL); - } - } - - return jt; + return subprog->jt; } /* "conditional jump with N edges" */ @@ -381,27 +412,39 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) struct bpf_iarray *jt; int i, w; - jt = env->insn_aux_data[t].jt; - if (!jt) { - jt = create_jt(t, env); - if (IS_ERR(jt)) - return PTR_ERR(jt); + /* + * The first visit pushes every target that still needs exploring, and + * those are all done by the time this insn is revisited, so there is + * nothing left to walk the jump table for. + */ + if (insn_state[t] & BRANCH) + return DONE_EXPLORING; - env->insn_aux_data[t].jt = jt; + jt = subprog_jt(t, env); + if (IS_ERR(jt)) + return PTR_ERR(jt); + + insn_state[t] |= BRANCH; + + if (check_add_overflow(env->cfg.gotox_edges, jt->cnt, + &env->cfg.gotox_edges) || + env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) { + verbose(env, "number of indirect jump edges in the program exceeds %u\n", + BPF_MAX_GOTOX_EDGES); + bpf_diag_program_structure( + env, t, "too many indirect jump edges", + "Reduce the number of indirect jumps, or the number of distinct targets they can reach.", + "The program has more than %u indirect jump edges in total, counted over every gotox instruction.", + BPF_MAX_GOTOX_EDGES); + return -E2BIG; } mark_prune_point(env, t); for (i = 0; i < jt->cnt; i++) { w = jt->items[i]; - if (w < 0 || w >= env->prog->len) { - verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); - bpf_diag_program_structure( - env, t, "indirect jump out of range", - "Keep indirect jump targets inside the program.", - "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", - t, w, env->prog->len - 1); - return -EINVAL; - } + if (verifier_bug_if(w < 0 || w >= env->prog->len, env, + "indirect jump from insn %d to %d out of range", t, w)) + return -EFAULT; mark_jmp_point(env, w); mark_jump_target(env, w); @@ -422,27 +465,73 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) } /* - * Instructions that can abnormally return from a subprog (tail_call - * upon success, ld_{abs,ind} upon load failure) have a hidden exit - * that the verifier must account for. + * Return pointers to functions in the read-only map that ld_imm64 instruction + * 't' loads the address of, or of its value, if there are any. */ -static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) +static struct bpf_func_ptr *insn_func_ptrs(struct bpf_verifier_env *env, int t, u32 *cnt) { - struct bpf_subprog_info *subprog; - struct bpf_iarray *jt; + struct bpf_insn *insn = &env->prog->insnsi[t]; - if (env->insn_aux_data[t].jt) - return 0; + *cnt = 0; + if (!env->func_ptr_cnt || !bpf_is_ldimm64(insn)) + return NULL; + if (insn->src_reg != BPF_PSEUDO_MAP_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_IDX_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_FD && + insn->src_reg != BPF_PSEUDO_MAP_IDX) + return NULL; - jt = bpf_iarray_realloc(NULL, 2); - if (!jt) - return -ENOMEM; + return bpf_map_func_ptrs(env, env->used_maps[env->insn_aux_data[t].map_index], cnt); +} - subprog = bpf_find_containing_subprog(env, t); - jt->items[0] = t + 1; - jt->items[1] = subprog->exit_idx; - env->insn_aux_data[t].jt = jt; - return 0; +/* + * ld_imm64 that loads the address of a map that has pointers to functions + * is similar to ld_imm64 with BPF_PSEUDO_FUNC that loads the address of one + * function: any of them may be read from the map and called via callx later. + * Treat it as a call of all of them. + */ +static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env, + struct bpf_func_ptr *ptrs, u32 cnt) +{ + int *insn_stack = env->cfg.insn_stack; + int *insn_state = env->cfg.insn_state; + int ret, w; + u32 i; + + ret = push_insn(t, t + 2, FALLTHROUGH, env); + if (ret) + return ret; + + /* + * Explore one function at a time like push_insn() does, so that + * DISCOVERED function is a caller of 't' and not the one that waits in + * insn_stack[], which merge_callee_effects() relies on. + */ + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + if (insn_state[w]) + continue; + + if (env->cfg.cur_stack >= env->prog->len) + return -E2BIG; + + insn_stack[env->cfg.cur_stack++] = w; + insn_state[w] = DISCOVERED; + return KEEP_EXPLORING; + } + + /* all functions are explored, their effects are known */ + mark_prune_point(env, t); + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + merge_callee_effects(env, t, w); + + /* the same marks as push_insn() leaves on a branch target */ + mark_prune_point(env, w); + mark_jmp_point(env, w); + mark_jump_target(env, w); + } + return DONE_EXPLORING; } /* Visits the instruction at index t and returns one of the following: @@ -453,21 +542,20 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) static int visit_insn(int t, struct bpf_verifier_env *env) { struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; + struct bpf_func_ptr *ptrs; int ret, off, insn_sz; + u32 cnt; if (bpf_pseudo_func(insn)) return visit_func_call_insn(t, insns, env, true); + ptrs = insn_func_ptrs(env, t, &cnt); + if (ptrs) + return visit_func_ptrs_insn(t, env, ptrs, cnt); + /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { - if (BPF_CLASS(insn->code) == BPF_LD && - (BPF_MODE(insn->code) == BPF_ABS || - BPF_MODE(insn->code) == BPF_IND)) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; return push_insn(t, t + insn_sz, FALLTHROUGH, env); } @@ -510,13 +598,10 @@ static int visit_insn(int t, struct bpf_verifier_env *env) */ if (ret == 0 && fp->might_sleep) mark_subprog_might_sleep(env, t); - if (bpf_helper_changes_pkt_data(insn->imm)) + /* nobody knows yet whether the dynptr is that of an skb */ + if (bpf_helper_changes_pkt_data(insn->imm) || + insn->imm == BPF_FUNC_dynptr_write) mark_subprog_changes_pkt_data(env, t); - if (insn->imm == BPF_FUNC_tail_call) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { struct bpf_call_arg_meta meta; @@ -684,6 +769,11 @@ int bpf_check_cfg(struct bpf_verifier_env *env) return ret; } +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + /* * For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range * [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start) @@ -691,43 +781,52 @@ int bpf_check_cfg(struct bpf_verifier_env *env) */ int bpf_compute_postorder(struct bpf_verifier_env *env) { - u32 cur_postorder, i, top, stack_sz, s; - int *stack = NULL, *postorder = NULL, *state = NULL; - struct bpf_iarray *succ; + int *stack = NULL, *postorder = NULL, *postorder_nums = NULL; + int subprog_idx, stack_sz, cur, s, cur_postorder, start; + struct dfs_state *state = NULL; + struct bpf_iarray *succ = NULL; + postorder_nums = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - if (!postorder || !state || !stack) { + state = kvzalloc_objs(struct dfs_state, env->prog->len, GFP_KERNEL_ACCOUNT); + if (!postorder_nums || !postorder || !stack || !state) { + kvfree(postorder_nums); kvfree(postorder); - kvfree(state); kvfree(stack); + kvfree(state); return -ENOMEM; } cur_postorder = 0; - for (i = 0; i < env->subprog_cnt; i++) { - env->subprog_info[i].postorder_start = cur_postorder; - stack[0] = env->subprog_info[i].start; + for (subprog_idx = 0; subprog_idx < env->subprog_cnt; subprog_idx++) { + start = env->subprog_info[subprog_idx].start; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + stack[0] = start; stack_sz = 1; + state[start].traversed = true; do { - top = stack[stack_sz - 1]; - state[top] |= DISCOVERED; - if (state[top] & EXPLORED) { - postorder[cur_postorder++] = top; + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + postorder_nums[cur] = cur_postorder; + postorder[cur_postorder] = cur; + cur_postorder++; stack_sz--; continue; } - succ = bpf_insn_successors(env, top); - for (s = 0; s < succ->cnt; ++s) { - if (!state[succ->items[s]]) { - stack[stack_sz++] = succ->items[s]; - state[succ->items[s]] |= DISCOVERED; - } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + stack_sz++; + continue; } - state[top] |= EXPLORED; + state[cur].next_succ++; } while (stack_sz); } - env->subprog_info[i].postorder_start = cur_postorder; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + env->cfg.postorder_nums = postorder_nums; env->cfg.insn_postorder = postorder; env->cfg.cur_postorder = cur_postorder; kvfree(stack); @@ -749,7 +848,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) struct bpf_insn_aux_data *aux = env->insn_aux_data; const u32 insn_cnt = env->prog->len; int stack_sz, dfs_sz, err = 0; - u32 *stack, *pre, *low, *dfs; + u32 *stack, *pre, *low, *dfs, *dfs_pos; u32 i, j, t, w; u32 next_preorder_num; u32 next_scc_id; @@ -762,13 +861,16 @@ int bpf_compute_scc(struct bpf_verifier_env *env) * - 'stack' accumulates vertices in DFS order, see invariant comment below; * - 'pre[t] == p' => preorder number of vertex 't' is 'p'; * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n'; - * - 'dfs' DFS traversal stack, used to emulate explicit recursion. + * - 'dfs' DFS traversal stack, used to emulate explicit recursion; + * - 'dfs_pos[k] == j' => the frame 'dfs[k]' resumes visiting its + * successors at index 'j'. */ stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT); - if (!stack || !pre || !low || !dfs) { + dfs_pos = kvcalloc(insn_cnt, sizeof(*dfs_pos), GFP_KERNEL_ACCOUNT); + if (!stack || !pre || !low || !dfs || !dfs_pos) { err = -ENOMEM; goto exit; } @@ -851,6 +953,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) stack_sz = 0; dfs_sz = 1; dfs[0] = i; + dfs_pos[0] = 0; dfs_continue: while (dfs_sz) { w = dfs[dfs_sz - 1]; @@ -860,13 +963,22 @@ int bpf_compute_scc(struct bpf_verifier_env *env) next_preorder_num++; stack[stack_sz++] = w; } - /* Visit 'w' successors */ + /* Visit remaining 'w' successors */ succ = bpf_insn_successors(env, w); - for (j = 0; j < succ->cnt; ++j) { + for (j = dfs_pos[dfs_sz - 1]; j < succ->cnt; ++j) { if (pre[succ->items[j]]) { low[w] = min(low[w], low[succ->items[j]]); } else { - dfs[dfs_sz++] = succ->items[j]; + /* + * Once DFS for succ->items[j] is complete, + * pre[succ->items[j]] is non-zero, hence + * resuming at 'j' allows to follow the + * low[w] = min(...) update branch above. + */ + dfs_pos[dfs_sz - 1] = j; + dfs_pos[dfs_sz] = 0; + dfs[dfs_sz] = succ->items[j]; + dfs_sz++; goto dfs_continue; } } @@ -916,5 +1028,6 @@ int bpf_compute_scc(struct bpf_verifier_env *env) kvfree(pre); kvfree(low); kvfree(dfs); + kvfree(dfs_pos); return err; }
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 149672c..ce82963 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c
@@ -24,6 +24,34 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE); EXPORT_SYMBOL(cgroup_bpf_enabled_key); +struct cgroup_struct_ops { + u32 type_id; + void *cfi_stubs; + bool mult_rcu; +}; + +static struct cgroup_struct_ops cgroup_struct_ops[MAX_CGROUP_BPF_ATTACH_TYPE]; + +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_rcu) +{ + cgroup_struct_ops[atype].type_id = type_id; + cgroup_struct_ops[atype].cfi_stubs = cfi_stubs; + cgroup_struct_ops[atype].mult_rcu = mult_rcu; +} + +static enum cgroup_bpf_attach_type find_atype_by_struct_ops_id(u32 type_id) +{ + enum cgroup_bpf_attach_type atype; + + for (atype = 0; atype < MAX_CGROUP_BPF_ATTACH_TYPE; atype++) { + if (cgroup_bpf_is_struct_ops_atype(atype) && + cgroup_struct_ops[atype].type_id && + cgroup_struct_ops[atype].type_id == type_id) + return atype; + } + return CGROUP_BPF_ATTACH_TYPE_INVALID; +} + /* * cgroup bpf destruction makes heavy use of work items and there can be a lot * of concurrent destructions. Use a separate workqueue so that cgroup bpf @@ -306,6 +334,19 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], bpf_cgroup_storage_link(storages[stype], cgrp, attach_type); } +static void cgroup_struct_ops_link_detach_wake(struct bpf_cgroup_link *link, bool wake_poll) +{ + cgroup_put(link->cgroup); + link->cgroup = NULL; + + bpf_map_put(link->map); + /* READ_ONCE in cgroup_struct_ops_link_poll */ + WRITE_ONCE(link->map, NULL); + + if (wake_poll) + wake_up_interruptible_poll(&link->wait_hup, EPOLLHUP); +} + /* Called when bpf_cgroup_link is auto-detached from dying cgroup. * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It * doesn't free link memory, which will eventually be done by bpf_link's @@ -313,10 +354,96 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], */ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) { + if (link->map) { + cgroup_struct_ops_link_detach_wake(link, true); + return; + } + + if (link->link.prog->expected_attach_type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(link->link.prog); cgroup_put(link->cgroup); link->cgroup = NULL; } +static void bpf_cgroup_array_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct bpf_prog_array, rcu)); +} + +static void bpf_cgroup_array_free(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + if (!array || array == &bpf_empty_prog_array) + return; + if (cgroup_struct_ops[atype].mult_rcu) + /* RCU tasks trace grace period implies RCU grace period. */ + call_rcu_tasks_trace(&array->rcu, bpf_cgroup_array_free_rcu); + else + kfree_rcu(array, rcu); +} + +static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype) +{ + if (cgroup_bpf_is_struct_ops_atype(atype)) + return cgroup_struct_ops[atype].cfi_stubs; + return bpf_prog_dummy(); +} + +static int bpf_cgroup_array_length(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int cnt = 0; + + for (item = array->items; item->prog; item++) { + if (item->prog != bpf_cgroup_array_dummy(atype)) + cnt++; + } + + return cnt; +} + +static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, + __u32 __user *prog_ids, int cnt, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int i = 0; + u32 id; + + for (item = array->items; item->prog && i < cnt; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + + if (cgroup_bpf_is_struct_ops_atype(atype)) + id = bpf_struct_ops_id(item->kdata); + else + id = item->prog->aux->id; + + if (copy_to_user(prog_ids + i, &id, sizeof(id))) + return -EFAULT; + i++; + } + return item->prog ? -ENOSPC : 0; +} + +static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, + int index, enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + + for (item = array->items; item->prog; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + if (!index) { + WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype)); + return 0; + } + index--; + } + return -ENOENT; +} + /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data @@ -346,18 +473,15 @@ static void cgroup_bpf_release(struct work_struct *work) bpf_trampoline_unlink_cgroup_shim(pl->prog); bpf_prog_put(pl->prog); } - if (pl->link) { - if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP) - bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog); + if (pl->link) bpf_cgroup_link_auto_detach(pl->link); - } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); } old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } list_for_each_entry_safe(storage, stmp, storages, list_cg) { @@ -399,6 +523,37 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) return NULL; } +static bool prog_list_is_struct_ops(const struct bpf_prog_list *pl) +{ + return pl->link && pl->link->map; +} + +static void prog_list_init_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + if (prog_list_is_struct_ops(pl)) { + item->kdata = bpf_struct_ops_map_kdata(pl->link->map); + return; + } + + item->prog = prog_list_prog(pl); + bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage); +} + +static void prog_list_replace_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + if (prog_list_is_struct_ops(pl)) + WRITE_ONCE(item->kdata, bpf_struct_ops_map_kdata(pl->link->map)); + else + WRITE_ONCE(item->prog, pl->link->link.prog); +} + +static u32 prog_list_id(struct bpf_prog_list *pl) +{ + if (prog_list_is_struct_ops(pl)) + return pl->link->map->id; + return prog_list_prog(pl)->aux->id; +} + /* count number of elements in the list. * it's slow but the list cannot be long */ @@ -408,7 +563,7 @@ static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt) u32 cnt = 0; hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (preorder_cnt && (pl->flags & BPF_F_PREORDER)) (*preorder_cnt)++; @@ -482,7 +637,7 @@ static int compute_effective_progs(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (pl->flags & BPF_F_PREORDER) { @@ -492,9 +647,7 @@ static int compute_effective_progs(struct cgroup *cgrp, item = &progs->items[fstart]; fstart++; } - item->prog = prog_list_prog(pl); - bpf_cgroup_storages_assign(item->cgroup_storage, - pl->storage); + prog_list_init_item(pl, item); cnt++; } @@ -517,7 +670,7 @@ static void activate_effective_progs(struct cgroup *cgrp, /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } /** @@ -557,7 +710,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) return 0; cleanup: for (i = 0; i < NR; i++) - bpf_prog_array_free(arrays[i]); + bpf_cgroup_array_free(arrays[i], i); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -612,7 +765,7 @@ static int update_effective_progs(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } continue; @@ -631,7 +784,7 @@ static int update_effective_progs(struct cgroup *cgrp, css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } @@ -870,7 +1023,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, old_pl_flags = pl->flags; bpf_cgroup_storages_assign(old_storage, pl->storage); } else { - pl = kmalloc_obj(*pl); + pl = kzalloc_obj(*pl); if (!pl) { bpf_cgroup_storages_free(new_storage); return -ENOMEM; @@ -973,9 +1126,10 @@ static int effective_prog_pos(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) - continue; - + /* + * No detaching pl (NULL prog and link) is visible to the callers, + * so skip the check compute_effective_progs() needs. + */ if (pl->flags & BPF_F_PREORDER) { if (pl == target_pl) pos = bstart; @@ -1022,7 +1176,7 @@ static void replace_effective_prog(struct cgroup *cgrp, desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, pl->link->link.prog); + prog_list_replace_item(pl, item); } } @@ -1177,7 +1331,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, lockdep_is_held(&cgroup_mutex)); /* Remove the program from the array */ - WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos), + WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), "Failed to purge a prog from array at index %d", pos); } } @@ -1287,7 +1441,15 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query && prog_attach_flags) return -EINVAL; - if (type == BPF_LSM_CGROUP) { + if (type == BPF_STRUCT_OPS) { + u32 type_id = attr->query.type_id; + + atype = find_atype_by_struct_ops_id(type_id); + if (atype == CGROUP_BPF_ATTACH_TYPE_INVALID) + return -ENOENT; + from_atype = to_atype = atype; + flags = 0; + } else if (type == BPF_LSM_CGROUP) { if (!effective_query && attr->query.prog_cnt && prog_ids && !prog_attach_flags) return -EINVAL; @@ -1307,7 +1469,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - total_cnt += bpf_prog_array_length(effective); + total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); } @@ -1337,35 +1499,33 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - cnt = min_t(int, bpf_prog_array_length(effective), total_cnt); - ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt); + cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); + ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); } else { struct hlist_head *progs; struct bpf_prog_list *pl; - struct bpf_prog *prog; u32 id; progs = &cgrp->bpf.progs[atype]; cnt = min_t(int, prog_list_length(progs, NULL), total_cnt); i = 0; hlist_for_each_entry(pl, progs, node) { - prog = prog_list_prog(pl); - id = prog->aux->id; + id = prog_list_id(pl); if (copy_to_user(prog_ids + i, &id, sizeof(id))) return -EFAULT; + if (prog_attach_flags) { + flags = cgrp->bpf.flags[atype] | + (pl->flags & BPF_F_PREORDER); + if (copy_to_user(prog_attach_flags + i, + &flags, sizeof(flags))) + return -EFAULT; + } if (++i == cnt) break; } - if (prog_attach_flags) { - flags = cgrp->bpf.flags[atype]; - - for (i = 0; i < cnt; i++) - if (copy_to_user(prog_attach_flags + i, - &flags, sizeof(flags))) - return -EFAULT; + if (prog_attach_flags) prog_attach_flags += cnt; - } } prog_ids += cnt; @@ -2329,7 +2489,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2772,6 +2932,274 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = { const struct bpf_prog_ops cg_sockopt_prog_ops = { }; +static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *map; + struct cgroup *cgrp; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + cgroup_unlock(); + return 0; + } + + map = cg_link->map; + atype = bpf_struct_ops_map_cgroup_atype(map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) + break; + } + if (WARN_ON_ONCE(!pl)) { + cgroup_unlock(); + return -ENOENT; + } + + /* mark deleted so compute_effective_progs() skips it */ + pl->link = NULL; + if (update_effective_progs(cgrp, atype)) { + pl->link = cg_link; + purge_effective_progs(cgrp, pl, atype); + } + + hlist_del(&pl->node); + cgroup_struct_ops_link_detach_wake(cg_link, wake_poll); + cgrp->bpf.revisions[atype]++; + + kfree(pl); + static_branch_dec(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return 0; +} + +static int cgroup_struct_ops_link_detach(struct bpf_link *link) +{ + return __cgroup_struct_ops_link_detach(link, true); +} + +static void cgroup_struct_ops_link_dealloc(struct bpf_link *link) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + __cgroup_struct_ops_link_detach(link, false); + kfree(cg_link); +} + +static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) +{ + struct bpf_cgroup_link *cg_link = + container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return; + } + + seq_printf(seq, "map_id:\t%u\n", cg_link->map->id); + seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup)); + cgroup_unlock(); +} + +static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return 0; + } + + info->struct_ops.map_id = cg_link->map->id; + info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup); + cgroup_unlock(); + return 0; +} + +static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *new_map, + struct bpf_map *expected_old_map) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *old_map; + struct cgroup *cgrp; + bool found = false; + int err; + + if (!bpf_struct_ops_valid_to_reg(new_map)) + return -EINVAL; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + err = -ENOLINK; + goto out; + } + + old_map = cg_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) + goto out; + + atype = bpf_struct_ops_map_cgroup_atype(new_map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) { + found = true; + break; + } + } + if (!found) { + err = -ENOENT; + goto out; + } + + bpf_map_inc(new_map); + WRITE_ONCE(cg_link->map, new_map); + replace_effective_prog(cgrp, atype, pl); + bpf_map_put(old_map); + cgrp->bpf.revisions[atype]++; + +out: + cgroup_unlock(); + return err; +} + +static __poll_t cgroup_struct_ops_link_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_cgroup_link *link = file->private_data; + + poll_wait(file, &link->wait_hup, pts); + + return READ_ONCE(link->map) ? 0 : EPOLLHUP; +} + +static const struct bpf_link_ops cgroup_struct_ops_link_ops = { + .dealloc = cgroup_struct_ops_link_dealloc, + .detach = cgroup_struct_ops_link_detach, + .show_fdinfo = cgroup_struct_ops_link_show_fdinfo, + .fill_link_info = cgroup_struct_ops_link_fill_link_info, + .update_map = cgroup_struct_ops_link_update, + .poll = cgroup_struct_ops_link_poll, +}; + +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr) +{ + u32 flags = attr->link_create.flags; + u32 pl_flags = (flags & BPF_F_PREORDER) | BPF_F_ALLOW_MULTI; + enum cgroup_bpf_attach_type atype; + struct bpf_link_primer link_primer; + struct bpf_cgroup_link *link; + struct bpf_prog_list *pl = NULL; + struct hlist_head *progs; + struct cgroup *cgrp; + int err; + + if (flags & ~BPF_F_LINK_ATTACH_MASK) + return -EINVAL; + + /* + * Attaching struct_ops to cgroup is through link only. All relative + * position must be corresponding to a link id or fd. + */ + if (attr->link_create.cgroup.relative_fd && !(flags & BPF_F_LINK)) + return -EINVAL; + + link = kzalloc_obj(*link, GFP_USER); + if (!link) + return -ENOMEM; + + bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, + &cgroup_struct_ops_link_ops, NULL, + attr->link_create.attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) { + kfree(link); + return err; + } + + cgrp = cgroup_get_from_fd(attr->link_create.target_fd); + if (IS_ERR(cgrp)) { + err = PTR_ERR(cgrp); + goto cleanup; + } + + bpf_map_inc(map); + link->map = map; + link->cgroup = cgrp; + init_waitqueue_head(&link->wait_hup); + + atype = bpf_struct_ops_map_cgroup_atype(map); + progs = &cgrp->bpf.progs[atype]; + + cgroup_lock(); + + if (attr->link_create.cgroup.expected_revision && + attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) { + err = -ESTALE; + goto unlock; + } + + if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS) { + err = -E2BIG; + goto unlock; + } + + pl = kzalloc_obj(*pl); + if (!pl) { + err = -ENOMEM; + goto unlock; + } + + pl->link = link; + pl->flags = pl_flags; + cgrp->bpf.flags[atype] = BPF_F_ALLOW_MULTI; + + err = insert_pl_to_hlist(pl, progs, NULL, link, + flags | BPF_F_ALLOW_MULTI, attr->link_create.cgroup.relative_fd); + if (err) + goto unlock; + + err = update_effective_progs(cgrp, atype); + if (err) { + hlist_del(&pl->node); + goto unlock; + } + + cgrp->bpf.revisions[atype]++; + static_branch_inc(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return bpf_link_settle(&link_primer); + +unlock: + cgroup_unlock(); + +cleanup: + kfree(pl); + if (link->cgroup) { + cgroup_put(link->cgroup); + link->cgroup = NULL; + bpf_map_put(link->map); + link->map = NULL; + } + bpf_link_cleanup(&link_primer); + return err; +} + /* Common helpers for cgroup hooks. */ const struct bpf_func_proto * cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c index 86142cb..8ec5187d 100644 --- a/kernel/bpf/cnum.c +++ b/kernel/bpf/cnum.c
@@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include <linux/bits.h> +#include <linux/bpf_verifier.h> /* for imod() */ #define T 32 #include "cnum_defs.h" @@ -118,3 +119,38 @@ struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b) } return t; } + +/* Intersect 'in' with the set of integers defined by equation 'base + step * k'. */ +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step) +{ + s64 smin = cnum64_smin(in); + s64 smax = cnum64_smax(in); + s64 lo, hi; + u16 d; + + if (step <= 1 || cnum64_is_empty(in) || cnum64_srange_overflow(in)) + return in; + /* + * Round smin up to the next value congruent to 'base' modulo 'step', + * i.e. increase smin by d = (base - smin) mod step: + * + * |<---- d ---->| + * |-----------|=============|... + * base+step*k smin base+step*(k+1) + */ + d = imod(base - imod(smin, step), step); + if ((u64)smax - (u64)smin < d) + return CNUM64_EMPTY; + lo = smin + d; + /* + * Round smax down to the previous value congruent to 'base' modulo 'step', + * i.e. decrease smax by d = (smax - base) mod step: + * + * |<--- d --->| + * ...|===========|-------------| + * base+step*k smax base+step*(k+1) + */ + d = imod(imod(smax, step) - base, step); + hi = smax - d; + return cnum64_from_srange(lo, hi); +}
diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index a90e317..05b17a8 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h
@@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max) return (struct cnum_t){ .base = base, .size = size }; } -/* True if this cnum represents two unsigned ranges. */ +/* + * True if this cnum represents two unsigned ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(urange_overflow)(struct cnum_t cnum) { /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ @@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum) * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range * [0..UT_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_umin / cnum{T}_umax. */ ut FN(umin)(struct cnum_t cnum) { @@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum) } EXPORT_SYMBOL_GPL(FN(umax)); -/* True if this cnum represents two signed ranges. */ +/* + * True if this cnum represents two signed ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(srange_overflow)(struct cnum_t cnum) { return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); @@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum) * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range * [ST_MIN..ST_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_smin / cnum{T}_smax. */ st FN(smin)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MIN - : min((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base; } st FN(smax)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MAX - : max((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size); } /* @@ -181,11 +193,55 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) static inline struct cnum_t FN(normalize)(struct cnum_t cnum) { - if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + if (cnum.size == UT_MAX) cnum.base = 0; return cnum; } +/* + * Return a smallest arc containing both 'a' and 'b'. + * Break equal-size ties by choosing the smaller base. + */ +struct cnum_t FN(union)(struct cnum_t a, struct cnum_t b) +{ + struct cnum_t b1, ab, ba; + ut end; + + if (FN(is_empty)(a)) + return b; + if (FN(is_empty)(b)) + return a; + + /* + * Rotate so that a1.base == 0 and a1.end == a.size. + * Normalize b1 to preserve the full-circle representation. + */ + b1 = FN(normalize)((struct cnum_t){ b.base - a.base, b.size }); + end = max(a.size, (ut)(b1.base + b1.size)); + + if (FN(urange_overflow)(b1)) { + /* a1 reaches b1's main arc: together they cover the circle. */ + if (b1.base <= a.size) + return (struct cnum_t){ 0, UT_MAX }; + + /* Extend b1's tail through a1's end, then rotate back. */ + return FN(normalize)((struct cnum_t){ b.base, end - b1.base }); + } + + /* ab, rotated back, covers both nonwrapping arcs. */ + ab = (struct cnum_t){ a.base, end }; + if (b1.base <= a.size) + return FN(normalize)(ab); + + /* The arcs are disjoint; ba is the other possible covering arc. */ + ba = (struct cnum_t){ b.base, a.size - b1.base }; + if (ba.size < ab.size || + (ba.size == ab.size && ba.base < ab.base)) + ab = ba; + + return FN(normalize)(ab); +} + struct cnum_t FN(add)(struct cnum_t a, struct cnum_t b) { if (FN(is_empty)(a) || FN(is_empty)(b)) @@ -210,12 +266,7 @@ bool FN(is_empty)(struct cnum_t cnum) bool FN(contains)(struct cnum_t cnum, ut v) { - if (FN(is_empty)(cnum)) - return false; - if (FN(urange_overflow)(cnum)) - return v >= cnum.base || v <= (ut)cnum.base + cnum.size; - else - return v >= cnum.base && v <= (ut)cnum.base + cnum.size; + return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size; } bool FN(is_const)(struct cnum_t cnum)
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 7f1b300..f9edfde 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c
@@ -180,9 +180,12 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * bool is_ldsx = mode == BPF_MEMSX; int off = src->val + insn->off; u64 val = 0; + u32 cnt; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || off < 0 || off + size > map->value_size || + /* so are the addresses of functions that the map points to */ + bpf_map_range_func_ptrs(env, map, off, size, &cnt) || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; break; @@ -191,7 +194,8 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * dst->val = val; break; case BPF_JMP: - if (opcode != BPF_CALL) + /* both 'call imm' and 'callx reg' clobber caller saved registers */ + if (BPF_OP(insn->code) != BPF_CALL) break; process_call: for (r = BPF_REG_0; r <= BPF_REG_5; r++) @@ -353,7 +357,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) if (!bpf_insn_is_cond_jump(insn->code)) continue; - if (bpf_is_may_goto_insn(insn)) + if (BPF_OP(insn->code) == BPF_JCOND) continue; if (!(aux->const_reg_mask & BIT(insn->dst_reg))) @@ -394,5 +398,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) /* recompute postorder, since CFG has changed */ kvfree(env->cfg.insn_postorder); env->cfg.insn_postorder = NULL; + kvfree(env->cfg.postorder_nums); + env->cfg.postorder_nums = NULL; return bpf_compute_postorder(env); }
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7f11555..3030beb 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c
@@ -18,6 +18,7 @@ */ #include <uapi/linux/btf.h> +#include <linux/ctype.h> #include <linux/filter.h> #include <linux/sched/signal.h> #include <linux/skbuff.h> @@ -142,10 +143,6 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->dst_mutex); mutex_init(&fp->aux->st_ops_assoc_mutex); -#ifdef CONFIG_BPF_SYSCALL - bpf_prog_stream_init(fp); -#endif - return fp; } @@ -175,7 +172,6 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) } return prog; } -EXPORT_SYMBOL_GPL(bpf_prog_alloc); int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) { @@ -273,6 +269,7 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); fp->pages = pages; fp->aux->prog = fp; + bpf_prog_dev_bound_moved(fp); /* We keep fp->aux from fp_old around in the new * reallocated structure. @@ -289,6 +286,9 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, void __bpf_prog_free(struct bpf_prog *fp) { if (fp->aux) { +#ifdef CONFIG_BPF_SYSCALL + bpf_prog_stream_free(fp); +#endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); mutex_destroy(&fp->aux->st_ops_assoc_mutex); @@ -590,6 +590,10 @@ bpf_prog_ksym_set_name(struct bpf_prog *prog) prog->aux->func_info[prog->aux->func_idx].type_id); func_name = btf_name_by_offset(prog->aux->btf, type->name_off); snprintf(sym, (size_t)(end - sym), "_%s", func_name); + /* the name of a function of Rust is not an identifier */ + for (; *sym; sym++) + if (!isalnum(*sym) && *sym != '_' && *sym != '.') + *sym = '_'; return; } @@ -1555,8 +1559,7 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) * know whether fp here is the clone or the original. */ fp->aux->prog = fp; - if (fp->aux->offload) - fp->aux->offload->prog = fp; + bpf_prog_dev_bound_moved(fp); bpf_prog_clone_free(fp_other); } @@ -1677,7 +1680,6 @@ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) { return 0; } -EXPORT_SYMBOL_GPL(__bpf_call_base); /* All UAPI available opcodes. */ #define BPF_INSN_MAP(INSN_2, INSN_3) \ @@ -1840,6 +1842,7 @@ bool bpf_opcode_in_insntable(u8 code) [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, [BPF_JMP | BPF_JA | BPF_X] = true, + [BPF_JMP | BPF_CALL | BPF_X] = true, [BPF_JMP | BPF_JCOND] = true, }; #undef BPF_INSN_3_TBL @@ -2634,11 +2637,68 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) return select_interpreter; } +/* + * Whether a function has an insn that works with the arena base, which a JIT + * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on + * the final insns of the function: an access through an arena pointer, or + * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address + * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, + * which the JIT rebases, has a function model that says so. A load of an + * __arena struct member is rewritten with an immediate and needs none; + * dereferencing the result does. + */ +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + const struct btf_func_model *m; + int i, j; + + if (!prog->aux->arena) + return false; + + for (i = 0; i < prog->len; i++, insn++) { + switch (BPF_CLASS(insn->code)) { + case BPF_LDX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) + return true; + break; + case BPF_ST: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) + return true; + break; + case BPF_STX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) + return true; + break; + case BPF_ALU64: + if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && + insn->off == BPF_ADDR_SPACE_CAST) + return true; + break; + case BPF_JMP: + if (!bpf_pseudo_kfunc_call(insn)) + break; + m = bpf_jit_find_kfunc_model(prog, insn); + if (!m) + break; + for (j = 0; j < m->nr_args; j++) + if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG) + return true; + break; + } + } + return false; +} + static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; + prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog); if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); @@ -2742,7 +2802,6 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) { return __bpf_prog_select_runtime(NULL, fp, err); } -EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); static unsigned int __bpf_prog_ret1(const void *ctx, const struct bpf_insn *insn) @@ -2803,6 +2862,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); } +struct bpf_prog *bpf_prog_dummy(void) +{ + return &dummy_bpf_prog.prog; +} + int bpf_prog_array_length(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; @@ -3035,13 +3099,18 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, bool sleepable; u32 i; - sleepable = aux->prog->sleepable; + sleepable = bpf_prog_runs_sleepable(aux->prog); for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) map->ops->map_poke_untrack(map, aux); if (sleepable) atomic64_dec(&map->sleepable_refcnt); + /* + * The program that didn't load is not a user of the map. libbpf + * loads the program again to get the log of the verifier. + */ + cmpxchg(&map->user, (unsigned long)aux, 0); bpf_map_put(map); } } @@ -3073,31 +3142,12 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } -static void bpf_prog_free_deferred(struct work_struct *work) +static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; int i; - aux = container_of(work, struct bpf_prog_aux, work); -#ifdef CONFIG_BPF_SYSCALL - bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); - bpf_prog_stream_free(aux->prog); -#endif -#ifdef CONFIG_CGROUP_BPF - if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) - bpf_cgroup_atype_put(aux->cgroup_atype); -#endif - bpf_free_used_maps(aux); - bpf_free_used_btfs(aux); - bpf_prog_disassoc_struct_ops(aux->prog); - if (bpf_prog_is_dev_bound(aux)) - bpf_prog_dev_bound_destroy(aux->prog); -#ifdef CONFIG_PERF_EVENTS - if (aux->prog->has_callchain_buf) - put_callchain_buffers(); -#endif - if (aux->dst_trampoline) - bpf_trampoline_put(aux->dst_trampoline); + aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork); for (i = 0; i < aux->real_func_cnt; i++) { /* We can just unlink the subprog poke descriptor table as * it was originally linked to the main program and is also @@ -3114,6 +3164,44 @@ static void bpf_prog_free_deferred(struct work_struct *work) } } +static void bpf_prog_free_deferred(struct work_struct *work) +{ + struct bpf_prog_aux *aux; + bool ksyms; + + aux = container_of(work, struct bpf_prog_aux, work); + /* + * A prog that still runs when its last reference goes needs its symbols, + * by which its exception table is found. It has ended by now. + */ + ksyms = aux->real_func_cnt || !list_empty(&aux->ksym.lnode); + bpf_prog_kallsyms_del_all(aux->prog); +#ifdef CONFIG_BPF_SYSCALL + bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); +#endif +#ifdef CONFIG_CGROUP_BPF + if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) + bpf_cgroup_atype_put(aux->cgroup_atype); +#endif + bpf_free_used_maps(aux); + bpf_free_used_btfs(aux); + bpf_prog_disassoc_struct_ops(aux->prog); + if (bpf_prog_is_dev_bound(aux)) + bpf_prog_dev_bound_destroy(aux->prog); +#ifdef CONFIG_PERF_EVENTS + if (aux->prog->has_callchain_buf) + put_callchain_buffers(); +#endif + if (aux->dst_trampoline) + bpf_trampoline_put(aux->dst_trampoline); + /* The symbols are looked up under rcu_read_lock() */ + INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); + if (ksyms) + queue_rcu_work(system_percpu_wq, &aux->rwork); + else + bpf_prog_free_images(&aux->rwork.work); +} + void bpf_prog_free(struct bpf_prog *fp) { struct bpf_prog_aux *aux = fp->aux; @@ -3124,7 +3212,6 @@ void bpf_prog_free(struct bpf_prog *fp) INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } -EXPORT_SYMBOL_GPL(bpf_prog_free); /* RNG for unprivileged user space with separated state from prandom_u32(). */ static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); @@ -3301,6 +3388,117 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +/* Return TRUE if the JIT backend supports callx (indirect call) instruction. */ +bool __weak bpf_jit_supports_callx(void) +{ + return false; +} + +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + +/* unwind_goto and unwind_exit */ +bool __weak bpf_jit_supports_unwind(void) +{ + return false; +} + +/* + * How this arch places a by-value kfunc argument, or NULL for one that has + * not opted in and so only takes an argument of a single eightbyte, which + * every convention places in slot order. + */ +const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void) +{ + return NULL; +} + +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot) +{ + u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0; + bool on_stack = false; + + for (i = 0; i < fm->nr_args; i++) { + bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG; + u32 pos; + + nslots = btf_func_model_arg_slots(fm, i); + + if (align16 && abi->even_reg_align) + nregs_used = round_up(nregs_used, 2); + + if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) { + /* wholly in registers */ + pos = nregs_used; + nregs_used += nslots; + } else if (!on_stack && abi->split_at_boundary) { + /* the last registers hold what fits, the stack the rest */ + pos = nregs_used; + stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE; + nregs_used = abi->nr_arg_regs; + on_stack = true; + } else { + /* wholly on the stack */ + if (align16 && abi->even_stack_align) + stack_off = round_up(stack_off, 2 * BPF_REG_SIZE); + pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE; + stack_off += nslots * BPF_REG_SIZE; + if (!abi->backfill_after_stack) + on_stack = true; + } + + for (k = 0; k < nslots; k++) + pos_of_slot[slot + k] = pos + k; + slot += nslots; + } + + return slot; +} + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves) +{ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 nslots, n = 0, s, back; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + back = nslots; + + /* + * An argument is two eightbytes at most, so it frees one register at + * most and only one argument ever moves down. Its destination is + * still in use, so carry it in the scratch. Only a lower slot can + * take the one it leaves, so the walk reaches it first. + */ + for (s = nslots; s > 0; s--) { + u8 slot = s - 1, pos = pos_of_slot[slot]; + + if (pos == slot) + continue; + + if (pos < slot) { + moves[n].dst = BPF_JIT_ARG_TMP; + back = slot; + } else { + moves[n].dst = pos; + } + moves[n].src = slot; + n++; + } + + if (back < nslots) { + moves[n].dst = pos_of_slot[back]; + moves[n].src = BPF_JIT_ARG_TMP; + n++; + } + + return n; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; @@ -3321,6 +3519,12 @@ bool __weak bpf_jit_supports_arena(void) return false; } +/* Whether JIT takes BPF_REG_AX as the address of arena access */ +bool __weak bpf_jit_supports_arena_scalar(void) +{ + return false; +} + bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; @@ -3386,6 +3590,19 @@ bool __weak bpf_jit_supports_private_stack(void) return false; } +/* + * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes. + * Its prologue, epilogue and tail call sequences must encode such frame + * sizes and a private stack must be sized from the program's depth. The + * budget is only granted alongside bpf_jit_supports_subprog_tailcalls(), + * whose tail calls land before the target sets up its own frame; see + * bpf_prog_stack_limit(). + */ +bool __weak bpf_jit_supports_large_stack(void) +{ + return false; +} + void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) { }
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 5e59ab8..eb87131 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c
@@ -45,6 +45,7 @@ */ #define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */ +#define CPU_MAP_HELD_DEVS 2048 struct bpf_cpu_map_entry; struct bpf_cpu_map; @@ -74,6 +75,13 @@ struct bpf_cpu_map_entry { struct completion kthread_running; struct rcu_work free_work; + + /* + * Every frame and skb in the queue holds a reference to its device. + * The kthread keeps it until nothing is left in GRO. + */ + struct net_device *held[CPU_MAP_HELD_DEVS]; + u32 nr_held; }; struct bpf_cpu_map { @@ -129,9 +137,11 @@ static void __cpu_map_ring_cleanup(struct ptr_ring *ring) WARN_ON_ONCE(1); if (unlikely(__ptr_test_bit(0, &ptr))) { __ptr_clear_bit(0, &ptr); + dev_put(((struct sk_buff *)ptr)->dev); kfree_skb(ptr); continue; } + dev_put(((struct xdp_frame *)ptr)->dev_rx); xdp_return_frame(ptr); } } @@ -311,7 +321,7 @@ static int cpu_map_kthread_run(void *data) void *frames[CPUMAP_BATCH]; void *skbs[CPUMAP_BATCH]; u32 i, n, m; - bool empty; + bool empty, full; /* Release CPU reschedule checks */ if (__ptr_ring_empty(rcpu->queue)) { @@ -344,10 +354,12 @@ static int cpu_map_kthread_run(void *data) struct sk_buff *skb = f; __ptr_clear_bit(0, &skb); + rcpu->held[rcpu->nr_held++] = skb->dev; skbs[ret.skb_n++] = skb; continue; } + rcpu->held[rcpu->nr_held++] = ((struct xdp_frame *)f)->dev_rx; frames[ret.xdp_n++] = f; page = virt_to_page(f); @@ -398,10 +410,13 @@ static int cpu_map_kthread_run(void *data) /* Flush either every 64 packets or in case of empty ring */ packets += n; empty = __ptr_ring_empty(rcpu->queue); - if (packets >= NAPI_POLL_WEIGHT || empty) { - cpu_map_gro_flush(rcpu, empty); + full = empty || rcpu->nr_held > CPU_MAP_HELD_DEVS - CPUMAP_BATCH; + if (packets >= NAPI_POLL_WEIGHT || full) { + cpu_map_gro_flush(rcpu, full); packets = 0; } + while (full && rcpu->nr_held) + dev_put(rcpu->held[--rcpu->nr_held]); local_bh_enable(); /* resched point, may call do_softirq() */ } @@ -738,12 +753,14 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) for (i = 0; i < bq->count; i++) { struct xdp_frame *xdpf = bq->q[i]; - int err; - err = __ptr_ring_produce(q, xdpf); - if (err) { + if (__ptr_ring_full(q)) { drops++; xdp_return_frame_rx_napi(xdpf); + } else { + /* The device cannot go before the flush, from now on it can */ + dev_hold(xdpf->dev_rx); + __ptr_ring_produce(q, xdpf); } processed++; } @@ -807,11 +824,15 @@ int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, __skb_pull(skb, skb->mac_len); skb_set_redirected(skb, false); + dev_hold(skb->dev); __ptr_set_bit(0, &skb); ret = ptr_ring_produce(rcpu->queue, skb); - if (ret < 0) + if (ret < 0) { + __ptr_clear_bit(0, &skb); + dev_put(skb->dev); goto trace; + } wake_up_process(rcpu->kthread); trace:
diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 3f3fe24..69bc29d 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c
@@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2024 Meta, Inc */ #include <linux/bpf.h> -#include <linux/bpf_crypto.h> #include <linux/bpf_mem_alloc.h> #include <linux/btf.h> #include <linux/btf_ids.h> #include <linux/filter.h> -#include <linux/scatterlist.h> #include <linux/skbuff.h> -#include <crypto/skcipher.h> - -struct bpf_crypto_type_list { - const struct bpf_crypto_type *type; - struct list_head list; -}; +#include <crypto/aes-cbc.h> +#include <crypto/aes-ecb.h> /* BPF crypto initialization parameters struct */ /** @@ -36,94 +30,53 @@ struct bpf_crypto_params { u32 authsize; }; -static LIST_HEAD(bpf_crypto_types); -static DECLARE_RWSEM(bpf_crypto_types_sem); +enum bpf_crypto_algo_id { + BPF_ALGO_AES_CBC, + BPF_ALGO_AES_ECB, +}; + +static const struct { + const char *type_name; + const char *algo_name; + enum bpf_crypto_algo_id algo; +} bpf_crypto_algos[] = { + { "skcipher", "cbc(aes)", BPF_ALGO_AES_CBC }, + { "skcipher", "ecb(aes)", BPF_ALGO_AES_ECB }, +}; + +static bool bpf_crypto_find_algo(const struct bpf_crypto_params *params, + enum bpf_crypto_algo_id *id_ret) +{ + for (size_t i = 0; i < ARRAY_SIZE(bpf_crypto_algos); i++) { + if (strncmp(bpf_crypto_algos[i].type_name, params->type, + sizeof(params->type)) == 0 && + strncmp(bpf_crypto_algos[i].algo_name, params->algo, + sizeof(params->algo)) == 0) { + *id_ret = bpf_crypto_algos[i].algo; + return true; + } + } + return false; +} /** * struct bpf_crypto_ctx - refcounted BPF crypto context structure - * @type: The pointer to bpf crypto type - * @tfm: The pointer to instance of crypto API struct. - * @siv_len: Size of IV and state storage for cipher + * @key: The crypto key + * @algo: The crypto algorithm ID * @rcu: The RCU head used to free the crypto context with RCU safety. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. */ struct bpf_crypto_ctx { - const struct bpf_crypto_type *type; - void *tfm; - u32 siv_len; + union { + struct aes_key aes; + } key; + enum bpf_crypto_algo_id algo; struct rcu_head rcu; refcount_t usage; }; -int bpf_crypto_register_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -EBUSY; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (!strcmp(node->type->name, type->name)) - goto unlock; - } - - node = kmalloc_obj(*node); - err = -ENOMEM; - if (!node) - goto unlock; - - node->type = type; - list_add(&node->list, &bpf_crypto_types); - err = 0; - -unlock: - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_register_type); - -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -ENOENT; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, type->name)) - continue; - - list_del(&node->list); - kfree(node); - err = 0; - break; - } - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_unregister_type); - -static const struct bpf_crypto_type *bpf_crypto_get_type(const char *name) -{ - const struct bpf_crypto_type *type = ERR_PTR(-ENOENT); - struct bpf_crypto_type_list *node; - - down_read(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, name)) - continue; - - if (try_module_get(node->type->owner)) - type = node->type; - break; - } - up_read(&bpf_crypto_types_sem); - - return type; -} - __bpf_kfunc_start_defs(); /** @@ -132,93 +85,70 @@ __bpf_kfunc_start_defs(); * Allocates a crypto context that can be used, acquired, and released by * a BPF program. The crypto context returned by this function must either * be embedded in a map as a kptr, or freed with bpf_crypto_ctx_release(). - * As crypto API functions use GFP_KERNEL allocations, this function can - * only be used in sleepable BPF programs. + * As this uses a GFP_KERNEL allocation, this function can only be used in + * sleepable BPF programs. * * bpf_crypto_ctx_create() allocates memory for crypto context. * It may return NULL if no memory is available. * @params: pointer to struct bpf_crypto_params which contains all the * details needed to initialise crypto context. - * @params__sz: size of steuct bpf_crypto_params usef by bpf program - * @err: integer to store error code when NULL is returned. + * @params__sz: size of struct bpf_crypto_params used by bpf program + * @err_ret: integer to store error code when NULL is returned. */ __bpf_kfunc struct bpf_crypto_ctx * bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, - int *err) + int *err_ret) { - const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; + int err; if (!params || params__sz != sizeof(struct bpf_crypto_params) || params->reserved[0] || params->reserved[1]) { - *err = -EINVAL; + *err_ret = -EINVAL; return NULL; } - type = bpf_crypto_get_type(params->type); - if (IS_ERR(type)) { - *err = PTR_ERR(type); - return NULL; - } - - if (!type->has_algo(params->algo)) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - - if (!!params->authsize ^ !!type->setauthsize) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - if (!params->key_len || params->key_len > sizeof(params->key)) { - *err = -EINVAL; - goto err_module_put; + *err_ret = -EINVAL; + return NULL; } ctx = kzalloc_obj(*ctx); if (!ctx) { - *err = -ENOMEM; - goto err_module_put; + *err_ret = -ENOMEM; + return NULL; } - ctx->type = type; - ctx->tfm = type->alloc_tfm(params->algo); - if (IS_ERR(ctx->tfm)) { - *err = PTR_ERR(ctx->tfm); - goto err_free_ctx; + if (!bpf_crypto_find_algo(params, &ctx->algo)) { + err = -EOPNOTSUPP; + goto out; } - if (params->authsize) { - *err = type->setauthsize(ctx->tfm, params->authsize); - if (*err) - goto err_free_tfm; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + case BPF_ALGO_AES_ECB: + if (params->authsize) + err = -EOPNOTSUPP; + else + err = aes_preparekey(&ctx->key.aes, params->key, + params->key_len); + break; + default: + WARN_ON_ONCE(1); + err = -EOPNOTSUPP; + break; } - *err = type->setkey(ctx->tfm, params->key, params->key_len); - if (*err) - goto err_free_tfm; - - if (type->get_flags(ctx->tfm) & CRYPTO_TFM_NEED_KEY) { - *err = -EINVAL; - goto err_free_tfm; +out: + if (err) { + kfree_sensitive(ctx); + *err_ret = err; + return NULL; } - - ctx->siv_len = type->ivsize(ctx->tfm) + type->statesize(ctx->tfm); - refcount_set(&ctx->usage, 1); - + *err_ret = 0; return ctx; - -err_free_tfm: - type->free_tfm(ctx->tfm); -err_free_ctx: - kfree(ctx); -err_module_put: - module_put(type->owner); - - return NULL; } static void crypto_free_cb(struct rcu_head *head) @@ -226,9 +156,7 @@ static void crypto_free_cb(struct rcu_head *head) struct bpf_crypto_ctx *ctx; ctx = container_of(head, struct bpf_crypto_ctx, rcu); - ctx->type->free_tfm(ctx->tfm); - module_put(ctx->type->owner); - kfree(ctx); + kfree_sensitive(ctx); } /** @@ -268,27 +196,53 @@ __bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) } CFI_NOSEAL(bpf_crypto_ctx_release_dtor); +static int bpf_aes_cbc_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != AES_BLOCK_SIZE) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_cbc_decrypt(dst, src, src_len, iv, &ctx->key.aes); + else + aes_cbc_encrypt(dst, src, src_len, iv, &ctx->key.aes); + return 0; +} + +static int bpf_aes_ecb_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != 0) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_ecb_decrypt(dst, src, src_len, &ctx->key.aes); + else + aes_ecb_encrypt(dst, src, src_len, &ctx->key.aes); + return 0; +} + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, - const struct bpf_dynptr_kern *siv, + const struct bpf_dynptr_kern *iv, bool decrypt) { - u32 src_len, dst_len, siv_len; + u32 src_len, dst_len, iv_len; const u8 *psrc; u8 *pdst, *piv; - int err; if (__bpf_dynptr_is_rdonly(dst)) return -EINVAL; - siv_len = siv ? __bpf_dynptr_size(siv) : 0; + iv_len = iv ? __bpf_dynptr_size(iv) : 0; src_len = __bpf_dynptr_size(src); dst_len = __bpf_dynptr_size(dst); - if (!src_len || !dst_len || src_len > dst_len) - return -EINVAL; - - if (siv_len != ctx->siv_len) + if (!src_len || !dst_len) return -EINVAL; psrc = __bpf_dynptr_data(src, src_len); @@ -298,14 +252,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, if (!pdst) return -EINVAL; - piv = siv_len ? __bpf_dynptr_data_rw(siv, siv_len) : NULL; - if (siv_len && !piv) + piv = iv_len ? __bpf_dynptr_data_rw(iv, iv_len) : NULL; + if (iv_len && !piv) return -EINVAL; - err = decrypt ? ctx->type->decrypt(ctx->tfm, psrc, pdst, src_len, piv) - : ctx->type->encrypt(ctx->tfm, psrc, pdst, src_len, piv); - - return err; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + return bpf_aes_cbc_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + case BPF_ALGO_AES_ECB: + return bpf_aes_ecb_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + default: + return -EINVAL; + } } /** @@ -313,20 +273,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the encrypted data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Decrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, true); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, true); } /** @@ -334,20 +294,20 @@ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the plain data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Encrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_encrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, false); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, false); } __bpf_kfunc_end_defs();
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index dc7b859..1b8f0d7 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c
@@ -350,6 +350,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, xdp_convert_frame_to_buff(xdpf, &xdp); xdp.txq = &txq; + rxq.mem.type = xdpf->mem_type; xdp.rxq = &rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 8f64bfd..b520479 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c
@@ -790,8 +790,9 @@ static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, } } -static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, - const char *fmt, ...) +static __printf(4, 5) void bpf_diag_source(struct bpf_verifier_env *env, + u32 insn_idx, const char *label, + const char *fmt, ...) { struct bpf_diag_scratch *scratch; struct bpf_linfo_source *source_lines; @@ -956,6 +957,37 @@ const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_t } } +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_MEM_SIZE: + case ARG_CONST_MEM_SIZE: + return "an integer scalar length for this memory argument"; + case ARG_PTR_TO_CTX: + return "the original program context pointer or preserve it before modifying registers"; + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return "an integer scalar value for this argument, not a pointer or resource object"; + case ARG_PTR_TO_CONST_STR: + return "a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value"; + case ARG_PTR_TO_DYNPTR: + return "the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path"; + case ARG_PTR_TO_ALLOC_BTF_ID: + return "a pointer returned by the matching BPF object allocation path"; + case ARG_PTR_TO_REFCOUNTED_KPTR: + return "an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field"; + case ARG_PTR_TO_ITER: + return "the address of a stack iterator object for iterator new, next, and destroy calls"; + case ARG_PTR_TO_IRQ_FLAG: + return "the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave()"; + case ARG_PTR_TO_CTX_OUT: + return "the attach hook's own output argument, loaded directly from the program context"; + default: + return "a value with one of the accepted pointer or scalar types for this call"; + } +} + static const char *diag_arg_ordinal(int argno) { switch (argno) { @@ -1083,16 +1115,17 @@ void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, const char *constraint, *context; u32 depth; - if (env->cur_state->active_rcu_locks) - ctx_kind = BPF_DIAG_CONTEXT_RCU; - else if (env->cur_state->active_preempt_locks) - ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; - else if (env->cur_state->active_irq_id) - ctx_kind = BPF_DIAG_CONTEXT_IRQ; - else if (env->cur_state->active_locks) - ctx_kind = BPF_DIAG_CONTEXT_LOCK; - else - ctx_kind = BPF_DIAG_CONTEXT_NONE; + ctx_kind = BPF_DIAG_CONTEXT_NONE; + if (env->cur_state->in_sleepable) { + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + } depth = diag_context_depth(env, ctx_kind); opts = (struct bpf_diag_history_opts) { @@ -1531,8 +1564,7 @@ static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { event.mod.origin_valid = true; } - if (old_reg && new_reg && - (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + if (old_reg && new_reg && reason == BPF_DIAG_MOD_WRITE && !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && !event.mod.origin_valid && diag_mod_keeps_lineage(env, &event)) @@ -1564,9 +1596,9 @@ static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, return NULL; return &state->stack_arg_regs[target->stack_arg]; case BPF_DIAG_MOD_TARGET_STACK_SLOT: - if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + if (target->spi >= bpf_stack_nr_slots(state)) return NULL; - return &state->stack[target->spi].spilled_ptr; + return &bpf_stack_slot(state, target->spi)->spilled_ptr; default: return NULL; } @@ -1582,7 +1614,7 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta for (frame = 0; frame <= vstate->curframe; frame++) { struct bpf_func_state *state = vstate->frame[frame]; unsigned long start, end; - u32 nslots = state->allocated_stack / BPF_REG_SIZE; + u32 nslots = bpf_stack_nr_slots(state); int spi; start = (unsigned long)state->regs; @@ -1601,6 +1633,11 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta return true; } + /* + * Map the pointer back to its slot by address, which relies on + * the slots forming one contiguous array as bpf_stack_slot() + * indexes it. + */ start = (unsigned long)state->stack; end = (unsigned long)(state->stack + nslots); if (nslots && addr >= start && addr < end) {
diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index d1b7994..a4102fb 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h
@@ -51,6 +51,7 @@ const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state);
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 3ce8d74..889fc07 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c
@@ -350,7 +350,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (opcode == BPF_CALL) { char tmp[64]; - if (insn->src_reg == BPF_PSEUDO_CALL) { + if (BPF_SRC(insn->code) == BPF_X) { + verbose(cbs->private_data, "(%02x) callx r%d", + insn->code, insn->dst_reg); + } else if (insn->src_reg == BPF_PSEUDO_CALL) { verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, @@ -372,11 +375,16 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg == BPF_MAY_GOTO) { verbose(cbs->private_data, "(%02x) may_goto pc%+d", insn->code, insn->off); + } else if (insn->code == (BPF_JMP | BPF_JCOND) && + insn->src_reg == BPF_UNWIND_GOTO) { + verbose(cbs->private_data, "(%02x) unwind_goto pc%+d", + insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { verbose(cbs->private_data, "(%02x) gotol pc%+d", insn->code, insn->imm); } else if (insn->code == (BPF_JMP | BPF_EXIT)) { - verbose(cbs->private_data, "(%02x) exit", insn->code); + verbose(cbs->private_data, "(%02x) %s", insn->code, + insn->src_reg == BPF_UNWIND_EXIT ? "unwind_exit" : "exit"); } else if (BPF_SRC(insn->code) == BPF_X) { verbose(cbs->private_data, "(%02x) if %c%d %s %c%d goto pc%+d",
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d6f8352..f062ef7 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c
@@ -5,8 +5,6 @@ #include <linux/bpf_verifier.h> #include <linux/filter.h> #include <linux/vmalloc.h> -#include <linux/bsearch.h> -#include <linux/sort.h> #include <linux/perf_event.h> #include <linux/sched/signal.h> #include <net/xdp.h> @@ -46,6 +44,31 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } +/* + * The insn accesses arena through a number. JITs add the base of arena + * to the register as it is, so the access goes through the low 32 bits of + * the number in BPF_REG_AX. Registers of the program are not changed: the + * number may be compared or stored later, and on another path the register + * may be PTR_TO_ARENA. + * + * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate + * of st through a number is not blinded. + */ +static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) +{ + bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); + struct bpf_insn *patch = buf; + + *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); + *patch = *insn; + if (load) + patch->src_reg = BPF_REG_AX; + else + patch->dst_reg = BPF_REG_AX; + patch++; + return patch - buf; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -117,73 +140,26 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) return dst_reg; } -static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) -{ - const struct bpf_kfunc_desc *d0 = a; - const struct bpf_kfunc_desc *d1 = b; - - if (d0->imm != d1->imm) - return d0->imm < d1->imm ? -1 : 1; - if (d0->offset != d1->offset) - return d0->offset < d1->offset ? -1 : 1; - return 0; -} - const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) { - const struct bpf_kfunc_desc desc = { - .imm = insn->imm, - .offset = insn->off, - }; const struct bpf_kfunc_desc *res; struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - res = bsearch(&desc, tab->descs, tab->nr_descs, - sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); + if (insn->off < 0 || insn->off >= tab->nr_descs) + return NULL; - return res ? &res->func_model : NULL; -} - -static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) -{ - unsigned long call_imm; - + res = &tab->descs[insn->off]; if (bpf_jit_supports_far_kfunc_call()) { - call_imm = desc->func_id; - } else { - call_imm = BPF_CALL_IMM(desc->addr); - /* Check whether the relative offset overflows desc->imm */ - if ((unsigned long)(s32)call_imm != call_imm) { - verbose(env, "address of kernel func_id %u is out of range\n", - desc->func_id); - return -EINVAL; - } - } - desc->imm = call_imm; - return 0; -} - -static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) -{ - struct bpf_kfunc_desc_tab *tab; - int i, err; - - tab = env->prog->aux->kfunc_tab; - if (!tab) - return 0; - - for (i = 0; i < tab->nr_descs; i++) { - err = set_kfunc_desc_imm(env, &tab->descs[i]); - if (err) - return err; + if (res->func_id != insn->imm) + return NULL; + } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { + return NULL; } - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), - kfunc_desc_cmp_by_imm_off, NULL); - return 0; + return &res->func_model; } static int add_kfunc_in_insns(struct bpf_verifier_env *env, @@ -214,12 +190,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif +enum bpf_patch_mode { + BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ + BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ +}; + /* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn, + enum bpf_patch_mode mode) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -233,8 +216,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -245,16 +235,39 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + /* + * In BPF_PATCH_MOVE_TARGET mode the original insn is the last + * slot by construction. + */ + if (mode == BPF_PATCH_KEEP_TARGET && + !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is * actually the first instruction, so move it back. This also matches with the behavior * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new - * instruction. + * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, + * so the flag already sits where needed. */ - if (data[off + cnt - 1].indirect_target) { + if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { data[off].indirect_target = 1; data[off + cnt - 1].indirect_target = 0; } @@ -274,7 +287,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len } } -static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) +static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) { int i; @@ -282,7 +295,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) return; for (i = 0; i < env->insn_array_map_cnt; i++) - bpf_insn_array_adjust(env->insn_array_maps[i], off, len); + bpf_insn_array_adjust(env->insn_array_maps[i], first, len); } static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) @@ -293,7 +306,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); } -static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) +static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) { struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; int i, sz = prog->aux->size_poke_tab; @@ -301,7 +314,7 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) for (i = 0; i < sz; i++) { desc = &tab[i]; - if (desc->insn_idx <= off) + if (desc->insn_idx < first) continue; desc->insn_idx += len - 1; } @@ -320,11 +333,15 @@ static bool bpf_rewrite_must_abort(void) return false; } -struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) +static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len, + enum bpf_patch_mode mode) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + /* The first slot whose targets move. */ + u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; + struct bpf_insn original_insn; if (bpf_rewrite_must_abort()) return NULL; @@ -340,6 +357,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -348,13 +366,58 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); adjust_subprog_starts(env, off, len); - adjust_insn_arrays(env, off, len); - adjust_poke_descs(new_prog, off, len); + adjust_insn_arrays(env, first, len); + bpf_adjust_func_ptrs(env, off, len); + adjust_poke_descs(new_prog, first, len); return new_prog; } +struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len) +{ + return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); +} + +/* + * insn was moved down by delta insns inside its own patch. Operands relative + * to the pc that point in front of the old position did not move with it. + */ +static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) +{ + u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); + s64 off = insn->imm, off_min = S32_MIN; + bool is_imm = true; + + if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { + /* subprog that started at the old position starts with the patch */ + if (off >= 0) + return 0; + } else if ((class == BPF_JMP || class == BPF_JMP32) && + op != BPF_CALL && op != BPF_EXIT) { + if (insn->code != (BPF_JMP32 | BPF_JA)) { + off = insn->off; + off_min = S16_MIN; + is_imm = false; + } + /* jump to itself stays */ + if (off >= -1) + return 0; + } else { + return 0; + } + + off -= delta; + if (off < off_min) + return -ERANGE; + if (is_imm) + insn->imm = off; + else + insn->off = off; + return 0; +} + /* * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the * jump offset by 'delta'. @@ -440,6 +503,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, * in adjust_btf_func() - no need to adjust */ } + /* The exception callback is live, so it cannot sit inside the removed range. */ + if (env->exception_callback_subprog >= j) + env->exception_callback_subprog -= j - i; } else { /* convert i from "first prog to remove" to "first to adjust" */ if (env->subprog_info[i].start == off) @@ -517,9 +583,7 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, return 0; } -/* - * Clean up dynamically allocated fields of aux data for instructions [start, ...] - */ +/* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; @@ -527,9 +591,10 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) int i; for (i = start; i < end; i++) { - if (aux_data[i].jt) { - kvfree(aux_data[i].jt); - aux_data[i].jt = NULL; + if (aux_data[i].loop) { + kvfree(aux_data[i].loop->exits); + kvfree(aux_data[i].loop); + aux_data[i].loop = NULL; } } } @@ -552,6 +617,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (err) return err; + /* before subprogs are adjusted, since it looks at them */ + bpf_adjust_func_ptrs_after_remove(env, off, cnt); + err = adjust_subprog_starts_after_remove(env, off, cnt); if (err) return err; @@ -590,6 +658,7 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); struct bpf_insn *insn = env->prog->insnsi; const int insn_cnt = env->prog->len; int i; @@ -598,6 +667,20 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) if (!bpf_insn_is_cond_jump(insn->code)) continue; + if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) { + /* + * The callee never exits. Its unwind_exit needs + * unwind_goto all the same, and the cpu predicts that it + * returns to what follows. Don't let the removal of dead + * code put something else there. + */ + if (!aux_data[i + 1].seen) { + memcpy(insn + 1, &trap, sizeof(trap)); + aux_data[i + 1].seen = env->pass_cnt; + } + continue; + } + if (!aux_data[i + 1].seen) ja.off = insn->off; else if (!aux_data[i + 1 + insn->off].seen) @@ -612,12 +695,41 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) } } +/* + * The address of a function might be taken by the code that is not dead, + * while the function itself is dead code, since nothing calls it. Keep the first + * instruction of such function, so that the address remains valid and differs + * from the addresses of other functions. Make it a trap like + * sanitize_dead_code() does. + */ +static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; + int i, t; + + for (i = 0; i < insn_cnt; i++) { + if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) + continue; + t = i + insn[i].imm + 1; + if (aux_data[t].seen) + continue; + memcpy(insn + t, &trap, sizeof(trap)); + aux_data[t].zext_dst = false; + aux_data[t].seen = env->pass_cnt; + } +} + int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; int insn_cnt = env->prog->len; int i, err; + keep_funcs_with_addr_taken(env); + for (i = 0; i < insn_cnt; i++) { int j; @@ -638,6 +750,7 @@ int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) int bpf_opt_remove_nops(struct bpf_verifier_env *env) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; bool is_may_goto_0, is_ja; @@ -649,6 +762,8 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env) if (!is_may_goto_0 && !is_ja) continue; + if (aux[i].indirect_target) + continue; err = verifier_remove_insns(env, i, 1); if (err) @@ -758,11 +873,61 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, return 0; } +/* + * The prologue takes the context from R1, and it can clobber R2-R5. An + * extension gets the arguments of the function that it replaces. + */ +static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) +{ + const struct bpf_subprog_info *sub = &env->subprog_info[0]; + int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + + if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX) + return false; + + for (i = 1; i < cnt; i++) + if (sub->args[i].arg_type != ARG_SCALAR) + return false; + + return true; +} + /* convert load instructions that access fields of a context type into a * sequence of instructions that access fields of the underlying structure: * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */ +/* + * A load through an __arena pointer member of a kernel struct reads the kernel + * address of arena memory. Append the conversion to the arena pointer form, + * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to + * __arena arguments. A __nullable member keeps NULL as NULL, tested on the + * full kernel address, and so does a probed load, which reads as 0 when it + * faults. Returns the number of insns in @buf, 0 when the insn loads no such + * member, or an error when it does so on some paths only. + */ +static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, + struct bpf_insn *buf) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + int cnt = 0; + u32 base; + + if (!aux->arena_member) + return 0; + if (aux->non_arena_load) { + verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); + return -EINVAL; + } + + base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); + buf[cnt++] = *insn; + if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) + buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); + buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); + return cnt; +} + int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; @@ -791,7 +956,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -814,7 +980,20 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + if (env->prog->type == BPF_PROG_TYPE_EXT && + !ext_prog_takes_prologue(env)) { + verbose(env, + "Extension writing packet data must take ctx and scalars only\n"); + return -EINVAL; + } + if (env->prog->type == BPF_PROG_TYPE_EXT && + bpf_check_ext_prologue(&env->log, + env->prog->aux->dst_prog, + env->prog->aux->attach_btf_id)) + return -EINVAL; + env->prog->aux->has_prologue = true; + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; @@ -845,7 +1024,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) struct bpf_insn *patch = insn_buf; *patch++ = BPF_ST_NOSPEC(); - *patch++ = *insn; + *patch = *insn; + ret = bpf_adj_moved_insn(patch++, 1); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[i + delta].orig_idx); + return ret; + } cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -887,6 +1072,21 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; + } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && + insn->imm == BPF_LOAD_ACQ && + env->insn_aux_data[i + delta].arena_member) { + /* an acquire load of an __arena member is rewritten like a plain load */ + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { @@ -978,11 +1178,23 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; - continue; - } - if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, - "access to a fault prone pointer is not rewritten as a probed one")) + } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) { return -EFAULT; + } + + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + if (!cnt) + continue; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; continue; } @@ -1088,6 +1300,59 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } +/* + * Replace the offsets of functions with their addresses in the maps of + * the program, see resolve_func_ptrs() in verifier.c. The program must be + * the only user of such map. From now on no other program can use it, see + * bpf_map_claim(). + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_prog **func) +{ + unsigned long me = (unsigned long)prog->aux; + struct bpf_func_ptr *ptr; + int i, err, subprog; + u64 addr, new, *slot; + + for (i = 0; i < env->func_ptr_cnt; i++) { + ptr = &env->func_ptrs[i]; + + /* pointers are sorted by map */ + if ((!i || ptr->map != ptr[-1].map) && + cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { + verbose(env, "map '%s' is used by another program\n", ptr->map->name); + return -EBUSY; + } + + /* it's the address of the value of the map whatever the offset is */ + err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); + if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) + return -EFAULT; + slot = (u64 *)(unsigned long)(addr + ptr->map_off); + + /* + * The function that is dead code is removed and the pointer is + * NULL. The program may read it, but callx of it is dead code + * too, see check_func_callx(). + */ + new = 0; + if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { + subprog = bpf_find_subprog(env, ptr->xlated_off); + if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", + ptr->xlated_off)) + return -EFAULT; + new = (unsigned long)func[subprog]->bpf_func; + } + + /* the map is frozen and nothing else uses it */ + if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, + "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) + return -EFAULT; + WRITE_ONCE(*slot, new); + } + return 0; +} + static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1198,6 +1463,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i]->aux->arena = prog->aux->arena; + func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); func[i]->aux->used_maps = env->used_maps; func[i]->aux->used_map_cnt = env->used_map_cnt; num_exentries = 0; @@ -1216,12 +1482,15 @@ static int jit_subprogs(struct bpf_verifier_env *env) if (BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) num_exentries++; + if (bpf_is_unwind_goto(insn)) + num_exentries++; } func[i]->aux->num_exentries = num_exentries; func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; + func[i]->aux->can_unwind = env->subprog_info[i].can_unwind; func[i]->aux->token = prog->aux->token; if (!i) func[i]->aux->exception_boundary = env->seen_exception; @@ -1278,6 +1547,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) cond_resched(); } + /* the addresses of all functions are final */ + err = resolve_func_ptrs(env, prog, func); + if (err) + goto out_free; + /* * Cleanup func[i]->aux fields which aren't required * or can become invalid in future @@ -1364,8 +1638,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) struct bpf_prog *prog, *orig_prog; u32 *orig_subprog_starts; + /* all functions that the maps point to are removed as dead code */ if (env->subprog_cnt <= 1) - return 0; + return resolve_func_ptrs(env, env->prog, NULL); prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { @@ -1522,11 +1797,37 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat env->prog = prog; info[cnt + 1].start = info[cnt].start; info[cnt].start = prog->len - len + 1; + /* The patch ends with the exit. */ + info[cnt].exit_idx = prog->len - 1; env->subprog_cnt++; env->hidden_subprog_cnt++; return 0; } +/* + * Expand may_goto: load the count from the stack, jump to the target of + * may_goto when it is zero, then the tail that updates the count. + * Use gotol when the target is too far for 16-bit offset of a conditional jump. + */ +static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, + const struct bpf_insn *tail, int tail_cnt) +{ + int cnt = 0; + + /* Forward jump has to step over the tail */ + off = off >= 0 ? off + tail_cnt : off - 1; + + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); + if (off == (s16)off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); + } else { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); + insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); + } + memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); + return cnt + tail_cnt; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -1579,6 +1880,20 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); + if (env->insn_aux_data[i + delta].arena_scalar) { + cnt = arena_scalar_access(insn, insn_buf); + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + prog = new_prog; + env->prog = prog; + insn = prog->insnsi + i + delta; + goto next_insn; + } + /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || @@ -1804,6 +2119,18 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { int stack_off_cnt = -stack_depth - 16; + /* + * AX is used as an argument to pass in stack_off_cnt + * (to add to r10/fp), and also as the return value of + * the call to arch_bpf_timed_may_goto. + */ + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), + BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), + BPF_EMIT_CALL(arch_bpf_timed_may_goto), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), + }; /* * Two 8 byte slots, depth-16 stores the count, and @@ -1820,22 +2147,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * after subtraction, rinse and repeat. */ stack_depth_extra = 16; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); - /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. - */ - insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); - insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); - insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); - cnt = 7; + cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -1847,16 +2160,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } else if (bpf_is_may_goto_insn(insn)) { int stack_off = -stack_depth - 8; + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), + }; stack_depth_extra = 8; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); - cnt = 4; + cnt = may_goto_expand(insn_buf, insn->off, stack_off, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -1948,6 +2259,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (!bpf_allow_tail_call_in_subprogs(env)) prog->aux->stack_depth = MAX_BPF_STACK; prog->aux->max_pkt_offset = MAX_PACKET_OFF; + prog->aux->max_rdonly_access = U32_MAX; + prog->aux->max_rdwr_access = U32_MAX; + if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) + prog->aux->max_tp_access = U32_MAX; /* mark bpf_tail_call as different opcode to avoid * conditional branch in the interpreter for every normal @@ -2013,7 +2328,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } - if (insn->imm == BPF_FUNC_timer_set_callback) { + aux = &env->insn_aux_data[i + delta]; + if (aux->arg_prog) { /* The verifier will process callback_fn as many times as necessary * with different maps and the register states prepared by * set_timer_callback_state will be accurate. @@ -2028,7 +2344,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * bpf_timer_set_callback-ed will return -EINVAL. */ struct bpf_insn ld_addrs[2] = { - BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), + BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), }; insn_buf[0] = ld_addrs[0]; @@ -2256,7 +2572,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 2; } insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); - insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + insn_buf[cnt++] = BPF_ALU32_IMM(BPF_LSH, BPF_REG_2, 3); insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); @@ -2477,9 +2793,17 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) BPF_MAX_LOOPS); } /* Copy first actual insn to preserve it */ - insn_buf[cnt++] = env->prog->insnsi[subprog_start]; + insn_buf[cnt] = env->prog->insnsi[subprog_start]; + ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[subprog_start].orig_idx); + return ret; + } + cnt++; - new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog; @@ -2509,10 +2833,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } } - ret = sort_kfunc_descs_by_imm_off(env); - if (ret) - return ret; - return 0; } @@ -2686,4 +3006,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) return 0; } -
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 13a2356..bf78e09 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c
@@ -100,8 +100,10 @@ struct bpf_htab { struct percpu_counter pcount; atomic_t count; bool use_percpu_counter; + bool has_hash; u32 n_buckets; /* number of hash buckets */ u32 elem_size; /* size of each element in bytes */ + u32 elem_offset;/* offset of htab_elem in bytes */ u32 hashrnd; }; @@ -117,19 +119,46 @@ struct htab_elem { }; }; }; - union { - /* pointer to per-cpu pointer */ - void *ptr_to_pptr; - struct bpf_lru_node lru_node; - }; - u32 hash; char key[] __aligned(8); }; +struct htab_elem_hashed { + u32 hash __aligned(8); + struct htab_elem elem; +}; + +struct htab_elem_lru { + struct bpf_lru_node lru_node; + struct htab_elem_hashed; +}; + +/* + * Only for non-preallocated PCPU maps with small keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem. + */ +struct htab_elem_pcpu { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem elem; +}; + +/* + * Only for non-preallocated PCPU maps with large keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem_hashed. + */ +struct htab_elem_pcpu_hashed { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem_hashed; +}; + +static_assert(offsetof(struct htab_elem_pcpu_hashed, ptr_to_pptr) == + offsetof(struct htab_elem_pcpu, ptr_to_pptr)); + struct htab_btf_record { struct btf_record *record; struct btf *btf; - u32 key_size; + u32 value_offset; }; static inline bool htab_is_prealloc(const struct bpf_htab *htab) @@ -184,6 +213,71 @@ static inline bool is_fd_htab(const struct bpf_htab *htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } +/* + * u32 hash reads are always atomic. If the hash is elided, key comparisons + * must also be atomic to avoid false positive matches from torn key + * reads/writes on recycled elements. That is only possible when the + * key fits within a word. + */ +static __always_inline bool htab_key_size_needs_hash(u32 key_size) +{ + return key_size > sizeof(unsigned long); +} + +/* + * When hash is omitted, key reads and writes must be atomic to support lockless + * RCU readers. Zero-extend keys to the word size to support all sub-word key + * sizes, regardless of the size of the caller's key allocation. + */ +static __always_inline unsigned long htab_zero_extend_key(const void *key, + u32 key_size) +{ + unsigned long k = 0; + + memcpy(&k, key, min_t(u32, key_size, sizeof(k))); + return k; +} + +static bool htab_has_hash(const struct bpf_htab *htab) +{ + return htab->has_hash; +} + +static u32 htab_elem_hash(struct htab_elem *l) +{ + return container_of(l, struct htab_elem_hashed, elem)->hash; +} + +static void htab_elem_set_hash(struct htab_elem *l, u32 hash) +{ + container_of(l, struct htab_elem_hashed, elem)->hash = hash; +} + +static void *htab_elem_container(const struct bpf_htab *htab, struct htab_elem *l) +{ + return (void *)l - htab->elem_offset; +} + +static void *htab_elem_get_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l) +{ + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + return pcpu_elem->ptr_to_pptr; +} + +static void htab_elem_set_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l, + void *ptr) +{ + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + pcpu_elem->ptr_to_pptr = ptr; +} + +static struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l) +{ + return &container_of(l, struct htab_elem_lru, elem)->lru_node; +} + static inline void *htab_elem_value(struct htab_elem *l, u32 key_size) { return l->key + round_up(key_size, 8); @@ -207,7 +301,7 @@ static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) { - return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); + return htab->elems + i * (u64)htab->elem_size + htab->elem_offset; } /* Both percpu and fd htab support in-place update, so no need for @@ -301,16 +395,16 @@ static void htab_free_elems(struct bpf_htab *htab) * bucket_lock followed by lru_lock is not allowed. In such cases, * bucket_lock needs to be released first before acquiring lru_lock. */ -static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, - u32 hash) +static struct htab_elem_lru *prealloc_lru_pop(struct bpf_htab *htab, void *key, + u32 hash) { struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); - struct htab_elem *l; + struct htab_elem_lru *l; if (node) { bpf_map_inc_elem_count(&htab->map); - l = container_of(node, struct htab_elem, lru_node); - memcpy(l->key, key, htab->map.key_size); + l = container_of(node, struct htab_elem_lru, lru_node); + memcpy(l->elem.key, key, htab->map.key_size); return l; } @@ -350,8 +444,8 @@ static int prealloc_init(struct bpf_htab *htab) if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem, hash) - - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, hash) - + offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); else @@ -362,11 +456,12 @@ static int prealloc_init(struct bpf_htab *htab) if (htab_is_lru(htab)) bpf_lru_populate(&htab->lru, htab->elems, - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, lru_node), htab->elem_size, num_entries); else pcpu_freelist_populate(&htab->freelist, - htab->elems + offsetof(struct htab_elem, fnode), + htab->elems + htab->elem_offset + + offsetof(struct htab_elem, fnode), htab->elem_size, num_entries); return 0; @@ -454,7 +549,7 @@ static int htab_map_alloc_check(union bpf_attr *attr) return -EINVAL; if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE - - sizeof(struct htab_elem)) + sizeof(struct htab_elem_lru)) /* if key_size + value_size is bigger, the user space won't be * able to access the elements via bpf syscall. This check * also makes sure that the elem_size doesn't overflow and it's @@ -471,14 +566,11 @@ static int htab_map_alloc_check(union bpf_attr *attr) static void htab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; - struct htab_elem *elem = obj; - void *map_value; if (IS_ERR_OR_NULL(hrec->record)) return; - map_value = htab_elem_value(elem, hrec->key_size); - bpf_obj_free_fields(hrec->record, map_value); + bpf_obj_free_fields(hrec->record, obj + hrec->value_offset); } static void htab_pcpu_mem_dtor(void *obj, void *ctx) @@ -508,7 +600,7 @@ static void htab_dtor_ctx_free(void *ctx) } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, - void (*dtor)(void *, void *)) + void (*dtor)(void *, void *), u32 value_offset) { struct htab_btf_record *hrec; int err; @@ -520,7 +612,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, hrec = kzalloc_obj(*hrec); if (!hrec) return -ENOMEM; - hrec->key_size = map->key_size; + hrec->value_offset = value_offset; hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); @@ -555,9 +647,12 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, * populated in htab_map_alloc(), so it will always appear as NULL. */ if (htab_is_percpu(htab)) - return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, 0); else - return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, + htab->elem_offset + + offsetof(struct htab_elem, key) + + round_up(map->key_size, 8)); } static struct bpf_map *htab_map_alloc(union bpf_attr *attr) @@ -580,6 +675,10 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) bpf_map_init_from_attr(&htab->map, attr); + /* Avoid hash memory use and comparisons where unnecessary. */ + htab->has_hash = htab_is_lru(htab) || + htab_key_size_needs_hash(htab->map.key_size); + if (percpu_lru) { /* ensure each CPU's lru list has >=1 elements. * since we are at it, make each lru list has the same @@ -601,7 +700,17 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); - htab->elem_size = sizeof(struct htab_elem) + + if (htab_is_lru(htab)) + htab->elem_offset = offsetof(struct htab_elem_lru, elem); + else if (percpu && !prealloc) + htab->elem_offset = htab_has_hash(htab) ? + offsetof(struct htab_elem_pcpu_hashed, elem) : + offsetof(struct htab_elem_pcpu, elem); + else if (htab_has_hash(htab)) + htab->elem_offset = offsetof(struct htab_elem_hashed, elem); + + htab->elem_size = htab->elem_offset + + sizeof(struct htab_elem) + round_up(htab->map.key_size, 8); if (percpu) htab->elem_size += sizeof(void *); @@ -692,11 +801,18 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) return ERR_PTR(err); } -static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) +static __always_inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) { - if (likely(key_len % 4 == 0)) + const u32 *k = key; + + if (key_len == sizeof(u32)) + return jhash_1word(k[0], hashrnd); + else if (key_len == sizeof(u64)) + return jhash_2words(k[0], k[1], hashrnd); + else if (likely(key_len % 4 == 0)) return jhash2(key, key_len / 4, hashrnd); - return jhash(key, key_len, hashrnd); + else + return jhash(key, key_len, hashrnd); } static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) @@ -709,35 +825,57 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 return &__select_bucket(htab, hash)->head; } -/* this lookup function can only be called with bucket lock taken */ -static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, - void *key, u32 key_size) +static __always_inline struct hlist_nulls_node * +__lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) { struct hlist_nulls_node *n; struct htab_elem *l; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + if (elem_has_hash) { + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (htab_elem_hash(l) == hash && + !memcmp(&l->key, key, key_size)) + break; + } else { + unsigned long k = htab_zero_extend_key(key, key_size); - return NULL; + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (READ_ONCE(*(unsigned long *)l->key) == k) + break; + } + + return n; +} + +/* this lookup function can only be called with bucket lock taken */ +static __always_inline struct htab_elem * +lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) +{ + struct hlist_nulls_node *n; + + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (is_a_nulls(n)) + return NULL; + + return hlist_nulls_entry(n, struct htab_elem, hash_node); } /* can be called without bucket lock. it will repeat the loop in * the unlikely event when elements moved from one bucket into another * while link list is being walked */ -static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, - u32 hash, void *key, - u32 key_size, u32 n_buckets) +static __always_inline struct htab_elem * +lookup_nulls_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash, u32 n_buckets) { struct hlist_nulls_node *n; - struct htab_elem *l; again: - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (!is_a_nulls(n)) + return hlist_nulls_entry(n, struct htab_elem, hash_node); if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) goto again; @@ -745,31 +883,56 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, return NULL; } -/* Called from syscall or from eBPF program directly, so - * arguments have to match bpf_map_lookup_elem() exactly. - * The return value is adjusted by BPF instructions - * in htab_map_gen_lookup(). - */ -static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, + u32 key_size, bool elem_has_hash) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); struct hlist_nulls_head *head; struct htab_elem *l; - u32 hash, key_size; + u32 hash; WARN_ON_ONCE(!bpf_rcu_lock_held()); - key_size = map->key_size; - hash = htab_map_hash(key, key_size, htab->hashrnd); head = select_bucket(htab, hash); - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, elem_has_hash, + htab->n_buckets); return l; } +/* + * Called from syscall or from eBPF program directly, so + * arguments have to match bpf_map_lookup_elem() exactly. + * The return value is adjusted by BPF instructions + * in htab_map_gen_lookup(). u32/u64 specializations + * allow utilizing compile-time key_size for optimization. + */ +static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +{ + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + + return __htab_lookup(map, key, map->key_size, htab_has_hash(htab)); +} + +/* + * Specialized lookups for htab_map_gen_lookup(). These are only used for + * BPF_MAP_TYPE_HASH (never LRU), so elem_has_hash depends only on key size. + */ +static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u32), + htab_key_size_needs_hash(sizeof(u32))); +} + +static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u64), + htab_key_size_needs_hash(sizeof(u64))); +} + static void *htab_map_lookup_elem(struct bpf_map *map, void *key) { struct htab_elem *l = __htab_map_lookup_elem(map, key); @@ -798,7 +961,16 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u32, + (void *(*)(struct bpf_map *map, void *key))NULL)); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u64, + (void *(*)(struct bpf_map *map, void *key))NULL)); + if (map->key_size == sizeof(u32)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u32); + else if (map->key_size == sizeof(u64)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u64); + else + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + @@ -813,7 +985,7 @@ static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, if (l) { if (mark) - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return htab_elem_value(l, map->key_size); } @@ -836,19 +1008,17 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn = insn_buf; const int ret = BPF_REG_0; const int ref_reg = BPF_REG_1; + const s16 ref_off = (int)offsetof(struct htab_elem_lru, lru_node) + + (int)offsetof(struct bpf_lru_node, ref) - + (int)offsetof(struct htab_elem_lru, elem); BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); - *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref)); + *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, ref_off); *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); - *insn++ = BPF_ST_MEM(BPF_B, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref), - 1); + *insn++ = BPF_ST_MEM(BPF_B, ret, ref_off, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + round_up(map->key_size, 8)); @@ -880,14 +1050,15 @@ static void check_and_cancel_fields(struct bpf_htab *htab, static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) { struct bpf_htab *htab = arg; - struct htab_elem *l = NULL, *tgt_l; + struct htab_elem_lru *tgt_l; + struct htab_elem *l = NULL; struct hlist_nulls_head *head; struct hlist_nulls_node *n; unsigned long flags; struct bucket *b; int ret; - tgt_l = container_of(node, struct htab_elem, lru_node); + tgt_l = container_of(node, struct htab_elem_lru, lru_node); b = __select_bucket(htab, tgt_l->hash); head = &b->head; @@ -896,7 +1067,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) return false; hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l == tgt_l) { + if (l == &tgt_l->elem) { hlist_nulls_del_rcu(&l->hash_node); bpf_map_dec_elem_count(&htab->map); break; @@ -904,9 +1075,9 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) htab_unlock_bucket(b, flags); - if (l == tgt_l) + if (l == &tgt_l->elem) check_and_cancel_fields(htab, l); - return l == tgt_l; + return l == &tgt_l->elem; } /* Called from syscall */ @@ -930,7 +1101,8 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) head = select_bucket(htab, hash); /* lookup the key */ - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); if (!l) goto find_first_elem; @@ -973,8 +1145,8 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); - bpf_mem_cache_free(&htab->ma, l); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(htab, l)); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l)); } static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) @@ -1107,11 +1279,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, if (prealloc) { if (old_elem) { /* if we're updating the existing element, - * use per-cpu extra elems to avoid freelist_pop/push + * use per-cpu extra elems to avoid freelist_pop/push. + * The caller puts old_elem into the slot once it is + * unlinked. Until then the slot is empty for an NMI + * or a kprobe that updates this map on this cpu. */ pl_new = this_cpu_ptr(htab->extra_elems); - l_new = *pl_new; - *pl_new = old_elem; + l_new = READ_ONCE(*pl_new); + do { + if (!l_new) + return ERR_PTR(-EBUSY); + } while (!try_cmpxchg_local(pl_new, &l_new, NULL)); } else { struct pcpu_freelist_node *l; @@ -1122,6 +1300,8 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, bpf_map_inc_elem_count(&htab->map); } } else { + void *container; + if (is_map_full(htab)) if (!old_elem) /* when map is full and update() is replacing @@ -1131,14 +1311,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, */ return ERR_PTR(-E2BIG); inc_elem_count(htab); - l_new = bpf_mem_cache_alloc(&htab->ma); - if (!l_new) { + container = bpf_mem_cache_alloc(&htab->ma); + if (!container) { l_new = ERR_PTR(-ENOMEM); goto dec_count; } + l_new = container + htab->elem_offset; } - memcpy(l_new->key, key, key_size); + if (htab_has_hash(htab)) + memcpy(l_new->key, key, key_size); + if (percpu) { if (prealloc) { pptr = htab_elem_get_ptr(l_new, key_size); @@ -1147,11 +1330,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); if (!ptr) { - bpf_mem_cache_free(&htab->ma, l_new); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l_new)); l_new = ERR_PTR(-ENOMEM); goto dec_count; } - l_new->ptr_to_pptr = ptr; + htab_elem_set_ptr_to_pptr(htab, l_new, ptr); pptr = *(void __percpu **)ptr; } @@ -1170,7 +1353,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); } - l_new->hash = hash; + if (htab_has_hash(htab)) + htab_elem_set_hash(l_new, hash); + else + WRITE_ONCE(*(unsigned long *)l_new->key, + htab_zero_extend_key(key, key_size)); return l_new; dec_count: dec_elem_count(htab); @@ -1180,11 +1367,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, u64 map_flags) { - if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) + if (l_old && (map_flags & BPF_NOEXIST)) /* elem already exists */ return -EEXIST; - if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) + if (!l_old && (map_flags & BPF_EXIST)) /* elem doesn't exist, cannot update it */ return -ENOENT; @@ -1221,6 +1408,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, return -EINVAL; /* find an element without taking the bucket lock */ l_old = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1242,7 +1430,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1277,11 +1465,15 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, cancel - * its reusable special fields before it is available for reuse. + /* + * l_old replaces the extra element that alloc_htab_elem() took. + * Cancel its reusable special fields first: once it is in the + * slot, the next update on this cpu can reuse it. */ - if (htab_is_prealloc(htab)) + if (htab_is_prealloc(htab)) { check_and_cancel_fields(htab, l_old); + WRITE_ONCE(*this_cpu_ptr(htab->extra_elems), l_old); + } } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) @@ -1296,14 +1488,15 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &elem->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem)); } static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new, *l_old = NULL; + struct htab_elem *l_old = NULL; + struct htab_elem_lru *l_new; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1331,13 +1524,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; - copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); + copy_map_value(&htab->map, htab_elem_value(&l_new->elem, map->key_size), value); ret = htab_lock_bucket(b, &flags); if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1346,7 +1539,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value /* add new element to the head of the list, so that * concurrent search will find it before old elem */ - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); if (l_old) { bpf_lru_node_set_ref(&l_new->lru_node); hlist_nulls_del_rcu(&l_old->hash_node); @@ -1358,7 +1551,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value err_lock_bucket: if (ret) - htab_lru_push_free(htab, l_new); + htab_lru_push_free(htab, &l_new->elem); else if (l_old) htab_lru_push_free(htab, l_old); @@ -1367,9 +1560,11 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value static int htab_map_check_update_flags(bool onallcpus, u64 map_flags) { + if (unlikely((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS)) + return -EINVAL; if (unlikely(!onallcpus && map_flags > BPF_EXIST)) return -EINVAL; - if (unlikely(onallcpus && ((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))) + if (unlikely(onallcpus && ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; return 0; } @@ -1404,7 +1599,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1442,7 +1637,8 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, bool onallcpus) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new = NULL, *l_old; + struct htab_elem_lru *l_new = NULL; + struct htab_elem *l_old; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1467,7 +1663,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, * to remove older elem from htab and this removal * operation will need a bucket lock. */ - if (map_flags != BPF_EXIST) { + if (!(map_flags & BPF_EXIST)) { l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; @@ -1477,22 +1673,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) goto err; if (l_old) { - bpf_lru_node_set_ref(&l_old->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_old)); /* per-cpu hash map can update value in-place */ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), value, onallcpus, map_flags); } else { - pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), + pcpu_init_value(htab, htab_elem_get_ptr(&l_new->elem, key_size), value, onallcpus, map_flags); - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); l_new = NULL; } ret = 0; @@ -1542,7 +1738,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (l) hlist_nulls_del_rcu(&l->hash_node); else @@ -1577,7 +1773,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, true); if (l) hlist_nulls_del_rcu(&l->hash_node); @@ -1718,7 +1914,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (!l) { ret = -ENOENT; goto out_unlock; @@ -2489,7 +2685,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) struct htab_elem *l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); } @@ -2505,7 +2701,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); } @@ -2904,18 +3100,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_mem_dtor(void *obj, void *ctx) -{ - struct htab_btf_record *hrec = ctx; - struct rhtab_elem *elem = obj; - - if (IS_ERR_OR_NULL(hrec->record)) - return; - - bpf_obj_free_fields(hrec->record, - rhtab_elem_value(elem, hrec->key_size)); -} - static void rhtab_free_elem(void *ptr, void *arg) { struct bpf_rhtab *rhtab = arg; @@ -3141,7 +3325,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, if (btf_type_is_void(key_type)) return -EINVAL; - return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); + return bpf_ma_set_dtor(map, &rhtab->ma, htab_mem_dtor, + offsetof(struct rhtab_elem, data) + + round_up(map->key_size, 8)); } static void rhtab_map_free_internal_structs(struct bpf_map *map)
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 712dca5..648abe1 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c
@@ -679,9 +679,44 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +static int __bpf_copy_from_user_mm(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm) +{ + int ret; + + ret = access_remote_vm(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret == size) + return 0; + + memset(dst, 0, size); + /* Return -EFAULT for partial read */ + return ret < 0 ? ret : -EFAULT; +} + +static int __bpf_copy_from_user_mm_str(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm, u64 flags) +{ + int ret; + + ret = copy_remote_mm_str(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret < 0) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, size); + return ret; + } + + if (flags & BPF_F_PAD_ZEROS) + memset(dst + ret, 0, size - ret); + + return ret + 1; +} + BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) { + struct mm_struct *mm; int ret; /* flags is not used yet */ @@ -691,13 +726,16 @@ BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, if (unlikely(!size)) return 0; - ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0); - if (ret == size) - return 0; + mm = get_task_mm(tsk); + if (!mm) { + memset(dst, 0, size); + return -EFAULT; + } - memset(dst, 0, size); - /* Return -EFAULT for partial read */ - return ret < 0 ? ret : -EFAULT; + ret = __bpf_copy_from_user_mm(dst, size, user_ptr, mm); + mmput(mm); + + return ret; } const struct bpf_func_proto bpf_copy_from_user_task_proto = { @@ -1092,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, @@ -1224,18 +1262,21 @@ static void bpf_wq_work(struct work_struct *work) BTF_TYPE_EMIT(struct bpf_wq); + /* keeps the program that callback_fn belongs to */ + rcu_read_lock_trace(); + callback_fn = READ_ONCE(cb->callback_fn); if (!callback_fn) - return; + goto out; key = map_key_from_value(map, value, &idx); - rcu_read_lock_trace(); - migrate_disable(); + migrate_disable(); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); migrate_enable(); +out: rcu_read_unlock_trace(); } @@ -1510,6 +1551,7 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_TIMER, .arg2_type = ARG_PTR_TO_FUNC, + .arg3_type = ARG_PTR_TO_PROG_AUX, }; static bool defer_timer_wq_op(void) @@ -3659,6 +3701,92 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user } /** + * bpf_copy_from_user_mm() - Copy data from an address space + * @dst: Destination address, in kernel space + * @dst__sz: Number of bytes to copy + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: Reserved for future use; must be zero + * + * Copies data from the user address space associated with @mm. The destination + * is zeroed if an attempted copy cannot be completed in full. Unsupported + * flags return -EINVAL without modifying @dst. + * + * Return: 0 on success, -EINVAL if @flags is non-zero, or -EFAULT if @mm is no + * longer active or the copy fails or is partial. + */ +__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags)) + return -EINVAL; + + if (unlikely(!dst__sz)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + memset(dst, 0, dst__sz); + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm(dst, dst__sz, unsafe_ptr__ign, mm); + mmput(mm); + + return ret; +} + +/** + * bpf_copy_from_user_mm_str() - Copy a string from an address space + * @dst: Destination address, in kernel space. This buffer must be + * at least @dst__sz bytes long + * @dst__sz: Maximum number of bytes to copy, including the trailing NUL + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: The only supported flag is BPF_F_PAD_ZEROS + * + * Copies a NUL-terminated string from the user address space associated with + * @mm. If the string is too long, @dst is still NUL-terminated unless @dst__sz + * is zero. + * + * If the flags are valid and BPF_F_PAD_ZEROS is set, the unused portion of + * @dst is cleared on success and all of @dst is cleared on a copy failure. + * Unsupported flags return -EINVAL without modifying @dst. + * + * Return: The number of copied bytes including the NUL terminator on success, + * -EINVAL if @flags is invalid, or -EFAULT if @mm is no longer active or the + * copy fails. + */ +__bpf_kfunc int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags & ~BPF_F_PAD_ZEROS)) + return -EINVAL; + + if (unlikely(dst__sz == 0)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, dst__sz); + else + *(char *)dst = '\0'; + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); + + return ret; +} + +/** * bpf_copy_from_user_task_str() - Copy a string from an task's address space * @dst: Destination address, in kernel space. This buffer must be * at least @dst__sz bytes long. @@ -3681,6 +3809,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, const void __user *unsafe_ptr__ign, struct task_struct *tsk, u64 flags) { + struct mm_struct *mm; int ret; if (unlikely(flags & ~BPF_F_PAD_ZEROS)) @@ -3689,17 +3818,20 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, if (unlikely(dst__sz == 0)) return 0; - ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0); - if (ret < 0) { + mm = get_task_mm(tsk); + if (!mm) { if (flags & BPF_F_PAD_ZEROS) memset(dst, 0, dst__sz); - return ret; + else + *(char *)dst = '\0'; + return -EFAULT; } - if (flags & BPF_F_PAD_ZEROS) - memset(dst + ret, 0, dst__sz - ret); + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); - return ret + 1; + return ret; } /* Keep unsigned long in prototype so that kfunc is usable when emitted to @@ -4437,15 +4569,21 @@ static void bpf_task_work_ctx_put(struct bpf_task_work_ctx *ctx) } } -static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx) +static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx, + struct task_struct *task) { /* * Scheduled task_work callback holds ctx ref, so if we successfully * cancelled, we put that ref on callback's behalf. If we couldn't * cancel, callback will inevitably run or has already completed * running, and it would have taken care of its ctx ref itself. + * + * The caller passes the task explicitly: the scheduling handler must + * not read ctx->task after the work is published (the callback may + * have reset it), while in the asynchronous cancellation path + * ctx->task is stable, see bpf_task_work_cancel_scheduled(). */ - if (task_work_cancel(ctx->task, &ctx->work)) + if (task_work_cancel(task, &ctx->work)) bpf_task_work_ctx_put(ctx); } @@ -4486,6 +4624,7 @@ static void bpf_task_work_callback(struct callback_head *cb) static void bpf_task_work_irq(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); + struct task_struct *task; enum bpf_task_work_state state; int err; @@ -4496,7 +4635,16 @@ static void bpf_task_work_irq(struct irq_work *irq_work) return; } - err = task_work_add(ctx->task, &ctx->work, ctx->mode); + /* + * Capture the task pointer before the work is published: once the + * callback runs, it may reset ctx->task. The capture happens while + * this round's task reference is held, and every path that can + * release it runs after this handler entered its rcu read-side + * section. bpf_task_release() is put_task_struct_rcu_user(), so the + * task_struct itself cannot be freed before this section ends. + */ + task = ctx->task; + err = task_work_add(task, &ctx->work, ctx->mode); if (err) { bpf_task_work_ctx_reset(ctx); /* @@ -4510,14 +4658,14 @@ static void bpf_task_work_irq(struct irq_work *irq_work) /* * It's technically possible for just scheduled task_work callback to - * complete running by now, going SCHEDULING -> RUNNING and then - * dropping its ctx refcount. Instead of capturing an extra ref just - * to protect below ctx->state access, we rely on rcu_read_lock - * above to prevent kfree_rcu from freeing ctx before we return. + * complete running by now, going SCHEDULING -> RUNNING, resetting + * ctx->task and dropping its ctx refcount. The rcu read-side section + * above keeps both the ctx memory and the captured task pointer + * valid until this handler returns. */ state = cmpxchg(&ctx->state, BPF_TW_SCHEDULING, BPF_TW_SCHEDULED); if (state == BPF_TW_FREED) - bpf_task_work_cancel(ctx); /* clean up if we switched into FREED state */ + bpf_task_work_cancel(ctx, task); /* clean up if we switched into FREED state */ } static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *tw, @@ -4537,6 +4685,7 @@ static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *t memset(ctx, 0, sizeof(*ctx)); refcount_set(&ctx->refcnt, 1); /* map's own ref */ ctx->state = BPF_TW_STANDBY; + init_irq_work(&ctx->irq_work, bpf_task_work_irq); old_ctx = cmpxchg(&twk->ctx, NULL, ctx); if (old_ctx) { @@ -4579,6 +4728,18 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work bpf_task_work_ctx_put(ctx); return ERR_PTR(-EBUSY); } + /* + * STANDBY can be published before the previous scheduling irq_work + * returns. Claim PENDING before checking BUSY to order the check after + * that publication. No irq_work has been queued for this attempt yet, + * so reject reuse and roll back to STANDBY while BUSY is set. Preserve + * FREED if deletion wins the rollback race. + */ + if (unlikely(irq_work_is_busy(&ctx->irq_work))) { + (void)cmpxchg(&ctx->state, BPF_TW_PENDING, BPF_TW_STANDBY); + bpf_task_work_ctx_put(ctx); + return ERR_PTR(-EBUSY); + } /* * If no process or bpffs is holding a reference to the map, no new callbacks should be @@ -4628,7 +4789,6 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work ctx->map = map; ctx->map_val = (void *)tw - map->record->task_work_off; init_task_work(&ctx->work, bpf_task_work_callback); - init_irq_work(&ctx->irq_work, bpf_task_work_irq); irq_work_queue(&ctx->irq_work); return 0; @@ -4676,6 +4836,106 @@ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct b return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } +typedef int (*bpf_rcu_callback_t)(struct bpf_map *map, void *key, void *value); + +/* Actual type for struct bpf_rcu_head */ +struct bpf_rcu_head_kern { + struct rcu_head rcu; + bpf_callback_t callback_fn; + struct bpf_map *map; + struct bpf_prog *prog; + u32 armed; +} __aligned(8); + +static void bpf_rcu_run_callback(struct rcu_head *rcu) +{ + struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu); + bpf_callback_t callback_fn = rh->callback_fn; + struct bpf_prog *prog = rh->prog; + struct bpf_map *map = rh->map; + void *value, *key; + u32 idx; + + value = (void *)rh - map->record->rcu_head_off; + key = map_key_from_value(map, value, &idx); + + /* Pairs with the arming cmpxchg(): rh may be re-armed as soon as this store lands. */ + smp_store_release(&rh->armed, 0); + + rcu_read_lock_dont_migrate(); + callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + rcu_read_unlock_migrate(); + + bpf_prog_put(prog); +} + +static int __bpf_call_rcu(struct bpf_rcu_head *rh, struct bpf_map *map, void *callback, + struct bpf_prog_aux *aux, bool trace) +{ + struct bpf_rcu_head_kern *rhk = (void *)rh; + struct bpf_prog *prog; + + BUILD_BUG_ON(sizeof(struct bpf_rcu_head_kern) > sizeof(struct bpf_rcu_head)); + BUILD_BUG_ON(__alignof__(struct bpf_rcu_head_kern) != __alignof__(struct bpf_rcu_head)); + BTF_TYPE_EMIT(struct bpf_rcu_head); + + /* A queued callback cannot be cancelled, so a self-rearming one would pin prog and map. */ + if (!atomic64_read(&map->usercnt)) + return -EPERM; + + if (cmpxchg(&rhk->armed, 0, 1)) + return -EBUSY; + + prog = bpf_prog_inc_not_zero(aux->prog); + if (IS_ERR(prog)) { + WRITE_ONCE(rhk->armed, 0); + return -EBADF; + } + + rhk->callback_fn = (bpf_callback_t)callback; + rhk->map = map; + rhk->prog = prog; + if (trace) + call_rcu_tasks_trace(&rhk->rcu, bpf_rcu_run_callback); + else + call_rcu(&rhk->rcu, bpf_rcu_run_callback); + return 0; +} + +/** + * bpf_call_rcu - Invoke a BPF callback after an RCU grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, false); +} + +/** + * bpf_call_rcu_tasks_trace - Invoke a BPF callback after an RCU tasks trace grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Waits for sleepable BPF programs too. The callback itself is not sleepable either way. + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu_tasks_trace(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, true); +} + static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, struct bpf_dynptr_kern *ptr) { @@ -4777,7 +5037,13 @@ static void bpf_task_work_cancel_scheduled(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); - bpf_task_work_cancel(ctx); /* this might put task_work callback's ref */ + /* + * Deletion observed SCHEDULED and won against the callback's + * transition to RUNNING. The callback takes its FREED exit without + * resetting ctx->task, and the transferred map reference prevents + * destruction until cancellation completes. + */ + bpf_task_work_cancel(ctx, ctx->task); /* this might put task_work callback's ref */ bpf_task_work_ctx_put(ctx); /* and here we put map's own ref that was transferred to us */ } @@ -4924,6 +5190,8 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW | KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm_str, KF_SLEEPABLE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) @@ -4969,6 +5237,8 @@ BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_ BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu_tasks_trace, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE) BTF_ID_FLAGS(func, bpf_timer_cancel_async)
diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 0000000..bffc356 --- /dev/null +++ b/kernel/bpf/keys.c
@@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include <linux/bpf.h> +#include <linux/cred.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/key.h> +#include <linux/moduleparam.h> +#include <linux/slab.h> + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init);
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 301fc60..e657e6d 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c
@@ -10,10 +10,40 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) -struct per_frame_masks { - spis_t may_read; /* stack slots that may be read by this instruction */ - spis_t must_write; /* stack slots written by this instruction */ - spis_t live_before; /* stack slots that may be read by this insn and its successors */ +/* + * Stack liveness is tracked with a 4-byte (half register) granularity. + * Half-slot 0 covers [fp-4, fp), half-slot 1 covers [fp-8, fp-4), and so on, + * hence FRAME_HALF_SPIS - 1 is the deepest half-slot a frame can have. + */ +#define FRAME_HALF_SPIS (MAX_BPF_STACK_JIT / BPF_HALF_REG_SIZE) +#define FRAME_MAX_WORDS BITS_TO_LONGS(FRAME_HALF_SPIS) + +/* Masks tracked for each instruction of a frame */ +enum { + FM_MAY_READ, /* stack slots that may be read by this instruction */ + FM_MUST_WRITE, /* stack slots written by this instruction */ + FM_LIVE_BEFORE, /* stack slots that may be read by this insn and its successors */ + FM_MAY_WRITE, /* stack slots that may be written by this instruction */ + FM_MASK_CNT, +}; + +/* + * Per instruction stack masks for one frame of a function instance. + * + * Most frames use only a fraction of the stack budget, so instead of masks + * wide enough for every half-slot of the largest possible frame, all masks + * of one array share the width @words, and the marking functions widen the + * array as deeper half-slots are recorded. Mask @kind of the instruction at + * relative index @i is at &bits[(i * FM_MASK_CNT + kind) * words]. A + * half-slot at or past @words * BITS_PER_LONG is never read by this frame, + * hence never live. An instruction that may read the whole frame, such as a + * call passing a frame pointer to another subprog, widens the array to the + * program's stack budget, the deepest an accepted program can reach, so + * that the read cannot lose half-slots to a later widening. + */ +struct frame_masks { + u32 words; + unsigned long bits[]; }; /* @@ -29,7 +59,7 @@ struct func_instance { u32 subprog_start; /* cached env->subprog_info[subprog].start */ u32 insn_cnt; /* cached number of insns in the function */ /* Per frame, per instruction masks, frames allocated lazily. */ - struct per_frame_masks *frames[MAX_CALL_FRAMES]; + struct frame_masks *frames[MAX_CALL_FRAMES]; bool must_write_initialized; }; @@ -151,53 +181,145 @@ static int relative_idx(struct func_instance *instance, u32 insn_idx) return insn_idx - instance->subprog_start; } -static struct per_frame_masks *get_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) +static u32 frame_mask_bits(struct frame_masks *fm) { - if (!instance->frames[frame]) + return fm->words * BITS_PER_LONG; +} + +static size_t frame_mask_words(struct func_instance *instance, u32 words) +{ + return (size_t)instance->insn_cnt * FM_MASK_CNT * words; +} + +/* Mask @kind of the instruction at relative index @rel */ +static unsigned long *rel_mask(struct frame_masks *fm, u32 rel, u32 kind) +{ + return fm->bits + ((size_t)rel * FM_MASK_CNT + kind) * fm->words; +} + +/* + * Make sure @frame has a mask array at least @words wide, allocating it or + * copying the existing masks into the wider stride as needed. + * @words must be in range [1, FRAME_MAX_WORDS]. + */ +static struct frame_masks *widen_frame_masks(struct func_instance *instance, + u32 frame, u32 words) +{ + struct frame_masks *old = instance->frames[frame], *new; + u32 i, kind; + + if (old && old->words >= words) + return old; + new = kvzalloc_flex(*new, bits, frame_mask_words(instance, words), GFP_KERNEL_ACCOUNT); + if (!new) return NULL; - - return &instance->frames[frame][relative_idx(instance, insn_idx)]; -} - -static struct per_frame_masks *alloc_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) -{ - struct per_frame_masks *arr; - - if (!instance->frames[frame]) { - arr = kvzalloc_objs(*arr, instance->insn_cnt, - GFP_KERNEL_ACCOUNT); - instance->frames[frame] = arr; - if (!arr) - return ERR_PTR(-ENOMEM); + new->words = words; + if (old) { + for (i = 0; i < instance->insn_cnt; i++) + for (kind = 0; kind < FM_MASK_CNT; kind++) + memcpy(rel_mask(new, i, kind), rel_mask(old, i, kind), + old->words * sizeof(*old->bits)); + kvfree(old); } - return get_frame_masks(instance, frame, insn_idx); + instance->frames[frame] = new; + return new; } -/* Accumulate may_read masks for @frame at @insn_idx */ -static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* + * Set the inclusive half-slot range [lo, hi] in mask @kind of @frame at @insn_idx. + * An empty range, including one with a negative @hi as computed for a write + * that does not fully cover any half-slot, marks nothing. + */ +static int mark_stack_range(struct func_instance *instance, u32 frame, u32 insn_idx, + u32 kind, s32 lo, s32 hi) { - struct per_frame_masks *masks; + struct frame_masks *fm; - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->may_read = spis_or(masks->may_read, mask); + /* + * An access past the frame bottom is rejected by the main verifier + * pass later, liveness only has to avoid running off the masks. + */ + hi = min_t(s32, hi, FRAME_HALF_SPIS - 1); + if (lo > hi) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(hi + 1)); + if (!fm) + return -ENOMEM; + bitmap_set(rel_mask(fm, relative_idx(instance, insn_idx), kind), lo, hi - lo + 1); return 0; } -static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* Accumulate may_read for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) { - struct per_frame_masks *masks; + return mark_stack_range(instance, frame, insn_idx, FM_MAY_READ, lo, hi); +} - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->must_write = spis_or(masks->must_write, mask); +/* Accumulate must_write for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MUST_WRITE, lo, hi); +} + +static int mark_stack_may_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MAY_WRITE, lo, hi); +} + +/* + * Mark every half-slot of @frame as possibly read by @insn_idx. This widens + * the masks to the program's stack budget: a full read recorded at a narrower + * width would leave the bits added by a later widening clear and lose part of + * it. No verifier state holds a slot past the budget, so no liveness query + * reaches beyond it; an access past the budget in unreachable code may still + * widen a frame's masks further, which is harmless. + */ +static int mark_stack_read_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_read(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +static int mark_stack_may_write_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_may_write(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +/* Accumulate @src, a mask @src_words wide, into @kind mask of @frame at @insn_idx */ +static int mark_stack_mask(struct func_instance *instance, u32 frame, u32 insn_idx, u32 kind, + const unsigned long *src, u32 src_words) +{ + u32 nbits = src_words * BITS_PER_LONG; + struct frame_masks *fm; + unsigned long *dst; + u32 last, w; + + last = find_last_bit(src, nbits); + if (last == nbits) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1)); + if (!fm) + return -ENOMEM; + dst = rel_mask(fm, relative_idx(instance, insn_idx), kind); + /* @src has no bits set past @last, hence none past @fm->words either */ + src_words = min(src_words, fm->words); + for (w = 0; w < src_words; w++) + dst[w] |= src[w]; return 0; } +static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx, + const unsigned long *src, u32 src_words) +{ + return mark_stack_mask(instance, frame, insn_idx, FM_MAY_READ, src, src_words); +} + int bpf_jmp_offset(struct bpf_insn *insn) { u8 code = insn->code; @@ -248,14 +370,18 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) struct bpf_iarray *succ, *jt; int insn_sz; - jt = env->insn_aux_data[idx].jt; - if (unlikely(jt)) - return jt; - /* pre-allocated array of size up to 2; reset cnt, as it may have been used already */ succ = env->succ; succ->cnt = 0; + /* All gotox of a subprogram share its jump table, see compute_subprog_jts() */ + if (unlikely(insn_is_gotox(insn))) { + jt = bpf_find_containing_subprog(env, idx)->jt; + if (verifier_bug_if(!jt, env, "no jump table for gotox insn %u", idx)) + return succ; + return jt; + } + opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)]; insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; if (opcode_info->can_fallthrough) @@ -264,6 +390,20 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) if (opcode_info->can_jump) succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1; + /* + * tail_call upon success and ld_{abs,ind} upon load failure return + * from the subprogram, which is a hidden edge to its exit. A subprogram + * without an exit insn has no exit to model. + */ + if (unlikely((bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call) || + (BPF_CLASS(insn->code) == BPF_LD && + (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)))) { + u32 exit_idx = bpf_find_containing_subprog(env, idx)->exit_idx; + + if (exit_idx != U32_MAX) + succ->items[succ->cnt++] = exit_idx; + } + return succ; } @@ -272,33 +412,42 @@ __diag_pop(); static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { - spis_t new_before, new_after; - struct per_frame_masks *insn, *succ_insn; + unsigned long new_after[FRAME_MAX_WORDS] = {}; + unsigned long *may_read, *must_write, *live_before; + struct frame_masks *fm = instance->frames[frame]; + u32 rel = relative_idx(instance, insn_idx); struct bpf_iarray *succ; - u32 s; - bool changed; + bool changed = false; + u32 s, w; succ = bpf_insn_successors(env, insn_idx); if (succ->cnt == 0) return false; - changed = false; - insn = get_frame_masks(instance, frame, insn_idx); - new_before = SPIS_ZERO; - new_after = SPIS_ZERO; + /* All instructions of one frame array share the same mask width */ for (s = 0; s < succ->cnt; ++s) { - succ_insn = get_frame_masks(instance, frame, succ->items[s]); - new_after = spis_or(new_after, succ_insn->live_before); + unsigned long *succ_live; + + succ_live = rel_mask(fm, relative_idx(instance, succ->items[s]), FM_LIVE_BEFORE); + for (w = 0; w < fm->words; w++) + new_after[w] |= succ_live[w]; } + may_read = rel_mask(fm, rel, FM_MAY_READ); + must_write = rel_mask(fm, rel, FM_MUST_WRITE); + live_before = rel_mask(fm, rel, FM_LIVE_BEFORE); /* * New "live_before" is a union of all "live_before" of successors * minus slots written by instruction plus slots read by instruction. * new_before = (new_after & ~insn->must_write) | insn->may_read */ - new_before = spis_or(spis_and(new_after, spis_not(insn->must_write)), - insn->may_read); - changed |= !spis_equal(new_before, insn->live_before); - insn->live_before = new_before; + for (w = 0; w < fm->words; w++) { + unsigned long new_before = (new_after[w] & ~must_write[w]) | may_read[w]; + + if (new_before != live_before[w]) { + live_before[w] = new_before; + changed = true; + } + } return changed; } @@ -329,10 +478,12 @@ static void update_instance(struct bpf_verifier_env *env, struct func_instance * static bool is_live_before(struct func_instance *instance, u32 insn_idx, u32 frameno, u32 half_spi) { - struct per_frame_masks *masks; + struct frame_masks *fm = instance->frames[frameno]; - masks = get_frame_masks(instance, frameno, insn_idx); - return masks && spis_test_bit(masks->live_before, half_spi); + /* No recorded access reaches past the masks, so nothing there is live */ + if (!fm || half_spi >= frame_mask_bits(fm)) + return false; + return test_bit(half_spi, rel_mask(fm, relative_idx(instance, insn_idx), FM_LIVE_BEFORE)); } int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st) @@ -356,12 +507,29 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_ return 0; } +/* + * Stack slots of outer frames that the callee may read are accounted as read + * by the @callsite insn itself, i.e. they are live before @callsite and may be + * dead after it, when: + * - @callsite calls a callback, which may be called several times; + * - @callsite is callx. Its callee is not known when stack liveness is + * computed, so there is no func instance for it at @callsite. While + * the verifier is in the callee lookup_instance() finds its standalone + * instance that knows nothing about outer frames. What the callee may read + * there is recorded by record_call_access(). + */ +static bool callee_stack_access_at_callsite(struct bpf_verifier_env *env, u32 callsite) +{ + return bpf_calls_callback(env, callsite) || + bpf_is_callx(&env->prog->insnsi[callsite]); +} + bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi) { /* * Slot is alive if it is read before q->insn_idx in current func instance, * or if for some outer func instance: - * - alive before callsite if callsite calls callback, otherwise + * - alive before callsite if callsite calls callback or is callx, otherwise * - alive after callsite */ struct live_stack_query *q = &env->liveness->live_stack_query; @@ -394,7 +562,7 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp /* Get callsite from verifier state, not from instance callchain */ callsite = q->callsites[i]; - alive = bpf_calls_callback(env, callsite) + alive = callee_stack_access_at_callsite(env, callsite) ? is_live_before(instance, callsite, rel, half_spi) : is_live_before(instance, callsite + 1, rel, half_spi); if (alive) @@ -430,17 +598,19 @@ static int spi_off(int spi) * When only one half is set, print as "-4h","-8h",... * Runs of 3+ consecutive fully-set SPIs are collapsed: "fp0-8..-24" */ -static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, spis_t spis) +static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, + const unsigned long *spis, u32 words) { int buf_sz = sizeof(env->tmp_str_buf); + int spi_cnt = words * BITS_PER_LONG / 2; char *buf = env->tmp_str_buf; int spi, n, run_start; buf[0] = '\0'; - for (spi = 0; spi < STACK_SLOTS / 2 && buf_sz > 0; spi++) { - bool lo = spis_test_bit(spis, spi * 2); - bool hi = spis_test_bit(spis, spi * 2 + 1); + for (spi = 0; spi < spi_cnt && buf_sz > 0; spi++) { + bool lo = test_bit(spi * 2, spis); + bool hi = test_bit(spi * 2 + 1, spis); const char *space = first ? "" : " "; if (!lo && !hi) @@ -450,16 +620,16 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, /* half-spi */ n = scnprintf(buf, buf_sz, "%sfp%d%d%s", space, frame, spi_off(spi) + (lo ? STACK_SLOT_SZ : 0), "h"); - } else if (spi + 2 < STACK_SLOTS / 2 && - spis_test_bit(spis, spi * 2 + 2) && - spis_test_bit(spis, spi * 2 + 3) && - spis_test_bit(spis, spi * 2 + 4) && - spis_test_bit(spis, spi * 2 + 5)) { + } else if (spi + 2 < spi_cnt && + test_bit(spi * 2 + 2, spis) && + test_bit(spi * 2 + 3, spis) && + test_bit(spi * 2 + 4, spis) && + test_bit(spi * 2 + 5, spis)) { /* 3+ consecutive full spis */ run_start = spi; - while (spi + 1 < STACK_SLOTS / 2 && - spis_test_bit(spis, (spi + 1) * 2) && - spis_test_bit(spis, (spi + 1) * 2 + 1)) + while (spi + 1 < spi_cnt && + test_bit((spi + 1) * 2, spis) && + test_bit((spi + 1) * 2 + 1, spis)) spi++; n = scnprintf(buf, buf_sz, "%sfp%d%d..%d", space, frame, spi_off(run_start), spi_off(spi)); @@ -474,15 +644,41 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, return env->tmp_str_buf; } +/* Print mask @kind of the instruction at relative index @i for every frame, if any bit is set. */ +static bool print_mask(struct bpf_verifier_env *env, struct func_instance *instance, int i, + const char *name, u32 kind) +{ + struct frame_masks *fm; + bool printed = false; + unsigned long *mask; + int frame; + u64 pos; + + pos = env->log.end_pos; + verbose(env, "%s", name); + for (frame = instance->depth; frame >= 0; --frame) { + fm = instance->frames[frame]; + if (!fm) + continue; + mask = rel_mask(fm, i, kind); + if (bitmap_empty(mask, frame_mask_bits(fm))) + continue; + verbose(env, "%s", fmt_spis_mask(env, frame, !printed, mask, fm->words)); + printed = true; + } + if (!printed) + bpf_vlog_reset(&env->log, pos); + return printed; +} + static void print_instance(struct bpf_verifier_env *env, struct func_instance *instance) { int start = env->subprog_info[instance->subprog].start; struct bpf_insn *insns = env->prog->insnsi; - struct per_frame_masks *masks; int len = instance->insn_cnt; - int insn_idx, frame, i; - bool has_use, has_def; u64 pos, insn_pos; + int insn_idx, i; + bool printed; if (!(env->log.level & BPF_LOG_LEVEL2)) return; @@ -491,35 +687,17 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i verbose(env, "%s:\n", fmt_instance(env, instance)); for (i = 0; i < len; i++) { insn_idx = start + i; - has_use = false; - has_def = false; pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); - pos = env->log.end_pos; - verbose(env, " use: "); - for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->may_read)) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, masks->may_read)); - has_use = true; - } - if (!has_use) - bpf_vlog_reset(&env->log, pos); - pos = env->log.end_pos; - verbose(env, " def: "); - for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->must_write)) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, masks->must_write)); - has_def = true; - } - if (!has_def) - bpf_vlog_reset(&env->log, has_use ? pos : insn_pos); + printed = false; + printed |= print_mask(env, instance, i, " use: ", FM_MAY_READ); + printed |= print_mask(env, instance, i, " def: ", FM_MUST_WRITE); + printed |= print_mask(env, instance, i, " may_def: ", FM_MAY_WRITE); + if (!printed) + bpf_vlog_reset(&env->log, insn_pos); verbose(env, "\n"); if (bpf_is_ldimm64(&insns[insn_idx])) i++; @@ -540,6 +718,49 @@ static int cmp_instances(const void *pa, const void *pb) return 0; } +/* OR the 8-byte slots touched by a half-slot (4-byte) mask into @slots. */ +static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask, u32 nbits) +{ + u32 slot; + + for (slot = 0; slot < MAX_BPF_STACK_SLOTS && slot * 2 + 1 < nbits; slot++) + if (test_bit(slot * 2, mask) || test_bit(slot * 2 + 1, mask)) + __set_bit(slot, slots); +} + +/* + * Precompute, for each instruction, the OR of may_write and live_before masks + * over its top frame across all func_instances reaching it, stash them in the + * insn_aux_data. + */ +static void compute_may_write_masks(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_liveness *liveness = env->liveness; + struct func_instance *instance; + struct frame_masks *fm; + u32 nbits; + int bkt, i; + + hash_for_each(liveness->func_instances, bkt, instance, hl_node) { + fm = instance->frames[instance->depth]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + for (i = 0; i < instance->insn_cnt; i++) { + half_spis_to_slots(aux[instance->subprog_start + i].may_write_mask, + rel_mask(fm, i, FM_MAY_WRITE), nbits); + half_spis_to_slots(aux[instance->subprog_start + i].live_stack_before, + rel_mask(fm, i, FM_LIVE_BEFORE), nbits); + } + } +} + +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx) +{ + return env->insn_aux_data[insn_idx].may_write_mask; +} + /* print use/def slots for all instances ordered by callsite first, then by depth */ static int print_instances(struct bpf_verifier_env *env) { @@ -584,9 +805,9 @@ static int print_instances(struct bpf_verifier_env *env) * - same frame + different offset -> offset-imprecise * - different frames -> fully-imprecise (bitmask OR) * - * At memory access sites (LDX/STX/ST), offset-imprecise marks only - * the known frame's access mask as SPIS_ALL, while fully-imprecise - * iterates bits in the bitmask and routes each frame to its target. + * At memory access sites (LDX/STX/ST), offset-imprecise marks the known + * frame as fully read, while fully-imprecise iterates bits in the bitmask + * and routes each frame to its target. */ #define MAX_ARG_OFFSETS 4 @@ -605,8 +826,15 @@ enum arg_track_state { ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */ }; -/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */ -#define MAX_ARG_SPILL_SLOTS 64 +/* + * The tracker follows as many spill slots as a subprog addresses directly + * through R10, see subprog_spill_slots(). A frame's slots at a call site are + * kept in a snapshot for the callees that receive pointers into that frame. + */ +struct spill_snapshot { + u32 slots; + struct arg_track at[]; +}; /* * Combined register + stack arg tracking: R0-R10 at indices 0-10, @@ -881,13 +1109,60 @@ static void arg_padd(struct arg_track *at, s64 delta) } /* - * Convert a byte offset from FP to a callee stack slot index. + * Slots the spill tracker may follow for a subprog of @len instructions + * without its per-instruction tables costing more than they could for the + * largest program while every frame stayed within MAX_BPF_STACK: as many + * entries as 64 slots need for BPF_COMPLEXITY_LIMIT_INSNS instructions. + */ +static u32 spill_slots_affordable(int len) +{ + u32 base = MAX_BPF_STACK / BPF_REG_SIZE; + + return max_t(u32, base, base * BPF_COMPLEXITY_LIMIT_INSNS / len); +} + +/* + * Number of 8-byte spill slots to track for the instructions in [@start, @end): + * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always + * followed, or the deepest 8-byte stack access made directly through R10 when + * that reaches further, up to the program's stack budget. Spills and fills are + * compiled as direct R10 accesses, so the count covers them whatever the frame + * size; a fill through a derived pointer that reaches past it is treated as + * imprecise, as one past the frame always was. + */ +static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end) +{ + struct bpf_insn *insns = env->prog->insnsi; + int deepest = 0; + + for (int i = start; i < end; i++) { + struct bpf_insn *insn = &insns[i]; + u8 class = BPF_CLASS(insn->code); + u8 base; + + if (BPF_SIZE(insn->code) != BPF_DW) + continue; + if (class == BPF_LDX) + base = insn->src_reg; + else if (class == BPF_STX || class == BPF_ST) + base = insn->dst_reg; + else + continue; + if (base == BPF_REG_FP && insn->off < 0) + deepest = max(deepest, -insn->off); + } + return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE, + env->stack_limit / BPF_REG_SIZE); +} + +/* + * Convert a byte offset from FP to one of the @nslots tracked stack slots. * Returns -1 if out of range or not 8-byte aligned. * Slot 0 = fp-8, slot 1 = fp-16, ..., slot 7 = fp-64, .... */ -static int fp_off_to_slot(s16 off) +static int fp_off_to_slot(s16 off, u32 nslots) { - if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8)) + if (off >= 0 || off < -(int)(nslots * 8)) return -1; if (off % 8) return -1; @@ -896,7 +1171,7 @@ static int fp_off_to_slot(s16 off) static struct arg_track fill_from_stack(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, + struct arg_track *at_stack_out, u32 nslots, int depth) { struct arg_track imp = { @@ -907,7 +1182,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); return slot >= 0 ? at_stack_out[slot] : imp; } @@ -920,7 +1195,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) return imp; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) return imp; result = __arg_track_join(result, at_stack_out[slot]); @@ -935,7 +1210,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, * When exact offset is unknown conservatively add reg values to all slots in at_stack_out. */ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, - int reg, struct arg_track *at_stack_out, + int reg, struct arg_track *at_stack_out, u32 nslots, struct arg_track *val, u32 sz) { struct arg_track none = { .frame = ARG_NONE }; @@ -943,7 +1218,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); if (slot >= 0) at_stack_out[slot] = new_val; @@ -951,7 +1226,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, } cnt = at_out[reg].off_cnt; if (cnt == 0) { - for (int slot = 0; slot < MAX_ARG_SPILL_SLOTS; slot++) + for (int slot = 0; slot < nslots; slot++) at_stack_out[slot] = __arg_track_join(at_stack_out[slot], new_val); return; } @@ -961,7 +1236,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) continue; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) continue; if (cnt == 1) @@ -975,16 +1250,17 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, * Clear all tracked callee stack slots overlapping the byte range * [off, off+sz-1] where off is a negative FP-relative offset. */ -static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt) +static void clear_overlapping_stack_slots(struct arg_track *at_stack, u32 nslots, s16 off, + u32 sz, int cnt) { struct arg_track none = { .frame = ARG_NONE }; if (cnt == 0) { - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) + for (int i = 0; i < nslots; i++) at_stack[i] = __arg_track_join(at_stack[i], none); return; } - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (int i = 0; i < nslots; i++) { int slot_start = -((i + 1) * 8); int slot_end = slot_start + 8; @@ -1002,33 +1278,33 @@ static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u */ static void clear_stack_for_all_offs(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, u32 sz) + struct arg_track *at_stack_out, u32 nslots, u32 sz) { int cnt, i; if (reg == BPF_REG_FP) { - clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1); + clear_overlapping_stack_slots(at_stack_out, nslots, insn->off, sz, 1); return; } cnt = at_out[reg].off_cnt; if (cnt == 0) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, cnt); return; } for (i = 0; i < cnt; i++) { s16 fp_off; if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, 0); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, 0); break; } - clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, fp_off, sz, cnt); } } static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, int idx, struct arg_track *at_in, struct arg_track *at_stack_in, - struct arg_track *at_out, struct arg_track *at_stack_out) + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots) { bool printed = false; int i; @@ -1060,7 +1336,7 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]); } - for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (i = 0; i < nslots; i++) { if (arg_track_eq(&at_stack_out[i], &at_stack_in[i])) continue; if (!printed) { @@ -1082,13 +1358,36 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) } /* + * Model 8-byte reads through @reg, including load-acquire + * and the old value fetched by atomic operations. + */ +static struct arg_track arg_track_deref(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct arg_track *at_out, int reg, + struct arg_track *at_stack_out, u32 nslots, + int depth, u32 *callsites) +{ + struct arg_track *ptr = &at_out[reg]; + + if (can_be_local_fp(depth, reg, ptr)) + return fill_from_stack(insn, at_out, reg, at_stack_out, nslots, depth); + if (ptr->frame >= 0 && ptr->frame < depth) { + struct spill_snapshot *parent = env->callsite_at_stack[callsites[ptr->frame]]; + + return fill_from_stack(insn, at_out, reg, parent->at, parent->slots, ptr->frame); + } + if (ptr->frame == ARG_IMPRECISE && !(ptr->mask & BIT(depth)) && ptr->mask) + return *ptr; + return (struct arg_track){ .frame = ARG_NONE }; +} + +/* * Pure dataflow transfer function for arg_track state. * Updates at_out[] based on how the instruction modifies registers. * Tracks spill/fill, but not other memory accesses. */ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, - struct arg_track *at_out, struct arg_track *at_stack_out, + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots, const struct arg_track *at_stack_arg_entry, struct func_instance *instance, u32 *callsites) @@ -1162,55 +1461,42 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, for (r = BPF_REG_0; r <= BPF_REG_5; r++) at_out[r] = none; } else if (class == BPF_LDX) { - u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src); - - /* - * Reload from callee stack: if src is current-frame FP-derived - * and the load is an 8-byte BPF_MEM, try to restore the spill - * identity. For imprecise sources fill_from_stack() returns - * ARG_IMPRECISE (off_cnt == 0). - */ - if (src_is_local_fp && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - *dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, depth); - } else if (src->frame >= 0 && src->frame < depth && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - struct arg_track *parent_stack = - env->callsite_at_stack[callsites[src->frame]]; - - *dst = fill_from_stack(insn, at_out, insn->src_reg, - parent_stack, src->frame); - } else if (src->frame == ARG_IMPRECISE && - !(src->mask & BIT(depth)) && src->mask && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - /* - * Imprecise src with only parent-frame bits: - * conservative fallback. - */ - *dst = *src; - } else { + /* Only an 8-byte BPF_MEM load can be a fill, see arg_track_deref(). */ + if (BPF_MODE(insn->code) == BPF_MEM && BPF_SIZE(insn->code) == BPF_DW) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else *dst = none; - } } else if (class == BPF_LD && BPF_MODE(insn->code) == BPF_IMM) { *dst = none; } else if (class == BPF_STX) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool dst_is_local_fp; + bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); + u8 mode = BPF_MODE(insn->code); - /* Track spills to current-frame FP-derived callee stack */ - dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); - if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM) - spill_to_stack(insn, at_out, insn->dst_reg, - at_stack_out, src, sz); - - if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (dst_is_local_fp && insn->imm != BPF_LOAD_ACQ) - clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + if (mode == BPF_MEM || (mode == BPF_ATOMIC && insn->imm == BPF_STORE_REL)) { + if (dst_is_local_fp) + spill_to_stack(insn, at_out, insn->dst_reg, + at_stack_out, nslots, src, sz); + } else if (mode == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { + if (sz == 8) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else + *dst = none; + } else if (mode == BPF_ATOMIC) { + /* BPF_XCHG, BPF_CMPXCHG or BPF_{ADD,AND,OR,XOR} | [BPF_FETCH] */ + struct arg_track old = none; r = bpf_atomic_load_reg(insn); + if (r >= 0 && sz == 8) + old = arg_track_deref(env, insn, at_out, insn->dst_reg, + at_stack_out, nslots, depth, callsites); + if (dst_is_local_fp) + clear_stack_for_all_offs(insn, at_out, insn->dst_reg, + at_stack_out, nslots, sz); if (r >= 0) - at_out[r] = none; + at_out[r] = old; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); @@ -1219,23 +1505,21 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, /* BPF_ST to FP-derived dst: clear overlapping stack slots */ if (dst_is_local_fp) clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + at_stack_out, nslots, sz); } } /* - * Record access_bytes from helper/kfunc or load/store insn. - * access_bytes > 0: stack read - * access_bytes < 0: stack write - * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read - * access_bytes == 0: no access - * + * Record possible reads and writes for every touched half-slot. A definite + * write requires full coverage, a known size, and a single possible offset. */ -static int record_stack_access_off(struct func_instance *instance, s64 fp_off, - s64 access_bytes, u32 frame, u32 insn_idx) +static int record_stack_access_off(struct func_instance *instance, const struct arg_track *arg, + s64 off_idx, struct arg_access_info info, + u32 frame, u32 insn_idx) { + s64 fp_off = arg->off[off_idx]; s32 slot_hi, slot_lo; - spis_t mask; + int err; if (fp_off >= 0) /* @@ -1244,68 +1528,68 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, * by the main verifier pass later. */ return 0; - if (access_bytes == S64_MIN) { - /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - mask = SPIS_ZERO; - spis_or_range(&mask, 0, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); + + /* + * Read and may_write marks include partially covered slots. + * An unknown access size may reach from fp_off to the frame top. + */ + slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; + slot_lo = info.size == U32_MAX + ? 0 + : max_t(s32, (-fp_off - info.size) / STACK_SLOT_SZ, 0); + if (info.may_read) { + err = mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; } - if (access_bytes > 0) { - /* Mark any touched slot as use */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0); - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); - } else if (access_bytes < 0) { + if (info.may_write) { + err = mark_stack_may_write(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; + } + if (info.must_write && info.size != U32_MAX && arg->off_cnt == 1) { /* Mark only fully covered slots as def */ - access_bytes = -access_bytes; slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; - slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); - if (slot_lo <= slot_hi) { - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_write(instance, frame, insn_idx, mask); - } + slot_lo = max_t(s32, (-fp_off - info.size + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); + return mark_stack_write(instance, frame, insn_idx, slot_lo, slot_hi); } return 0; } -/* - * 'arg' is FP-derived argument to helper/kfunc or load/store that - * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'. - */ -static int record_stack_access(struct func_instance *instance, +/* Record access through a pointer with a known frame and possibly known offsets. */ +static int record_stack_access(struct bpf_verifier_env *env, + struct func_instance *instance, const struct arg_track *arg, - s64 access_bytes, u32 frame, u32 insn_idx) + struct arg_access_info info, u32 frame, u32 insn_idx) { int i, err; - if (access_bytes == 0) + if (!info.size) return 0; if (arg->off_cnt == 0) { - if (access_bytes > 0 || access_bytes == S64_MIN) - return mark_stack_read(instance, frame, insn_idx, SPIS_ALL); + if (info.may_read) { + err = mark_stack_read_all(env, instance, frame, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, frame, insn_idx); + if (err) + return err; + } return 0; } - if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) - /* multi-offset write cannot set stack_def */ - return 0; - for (i = 0; i < arg->off_cnt; i++) { - err = record_stack_access_off(instance, arg->off[i], access_bytes, frame, insn_idx); + err = record_stack_access_off(instance, arg, i, info, frame, insn_idx); if (err) return err; } return 0; } -/* - * When a pointer is ARG_IMPRECISE, conservatively mark every frame in - * the bitmask as fully used. - */ -static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx) +/* Record possible effects on every candidate frame of an imprecise pointer. */ +static int record_imprecise(struct bpf_verifier_env *env, struct func_instance *instance, + struct arg_access_info info, u32 mask, u32 insn_idx) { int depth = instance->depth; int f, err; @@ -1314,9 +1598,16 @@ static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_i if (!(mask & 1)) continue; if (f <= depth) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); - if (err) - return err; + if (info.may_read) { + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, f, insn_idx); + if (err) + return err; + } } } return 0; @@ -1328,9 +1619,10 @@ static int record_load_store_access(struct bpf_verifier_env *env, struct arg_track *at, int insn_idx) { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + struct arg_access_info info = {}; int depth = instance->depth; - s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); u8 class = BPF_CLASS(insn->code); + bool read = false, write = false; struct arg_track resolved, *ptr; int oi; @@ -1347,23 +1639,34 @@ static int record_load_store_access(struct bpf_verifier_env *env, switch (class) { case BPF_LDX: ptr = &at[insn->src_reg]; + read = true; break; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (insn->imm == BPF_STORE_REL) - sz = -sz; - if (insn->imm == BPF_LOAD_ACQ) + switch (insn->imm) { + case BPF_LOAD_ACQ: ptr = &at[insn->src_reg]; - else + read = true; + break; + case BPF_STORE_REL: ptr = &at[insn->dst_reg]; + write = true; + break; + default: + /* ADD/AND/OR/XOR(+FETCH), XCHG, CMPXCHG */ + ptr = &at[insn->dst_reg]; + read = true; + write = true; + break; + } } else { ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; } break; case BPF_ST: ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; break; default: return 0; @@ -1382,10 +1685,14 @@ static int record_load_store_access(struct bpf_verifier_env *env, ptr = &resolved; } + info.may_read = read; + info.may_write = write; + info.must_write = write; + info.size = bpf_size_to_bytes(BPF_SIZE(insn->code)); if (ptr->frame >= 0 && ptr->frame <= depth) - return record_stack_access(instance, ptr, sz, ptr->frame, insn_idx); + return record_stack_access(env, instance, ptr, info, ptr->frame, insn_idx); if (ptr->frame == ARG_IMPRECISE) - return record_imprecise(instance, ptr->mask, insn_idx); + return record_imprecise(env, instance, info, ptr->mask, insn_idx); /* ARG_NONE: not derived from any frame pointer, skip */ return 0; } @@ -1396,37 +1703,42 @@ static int record_arg_access(struct bpf_verifier_env *env, struct arg_track *at, int arg_idx, int insn_idx) { + struct arg_access_info info; int depth = instance->depth; int frame = at->frame; int err = 0; - s64 bytes; if (!arg_is_fp(at)) return 0; if (bpf_helper_call(insn)) { - bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); } else if (bpf_pseudo_kfunc_call(insn)) { - bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + } else if (bpf_pseudo_call(insn)) { + info = bpf_global_subprog_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + err = mark_stack_may_write_all(env, instance, f, insn_idx); if (err) return err; } return 0; } - if (bytes == 0) + if (!info.size) return 0; if (frame >= 0 && frame <= depth) - err = record_stack_access(instance, at, bytes, frame, insn_idx); + err = record_stack_access(env, instance, at, info, frame, insn_idx); else if (frame == ARG_IMPRECISE) - err = record_imprecise(instance, at->mask, insn_idx); + err = record_imprecise(env, instance, info, at->mask, insn_idx); return err; } -/* Record stack access for a given 'at' state of helper/kfunc 'insn' */ +/* Record stack access for a given 'at' state of helper/kfunc/global subprog 'insn' */ static int record_call_access(struct bpf_verifier_env *env, struct func_instance *instance, struct arg_track *at, @@ -1434,21 +1746,36 @@ static int record_call_access(struct bpf_verifier_env *env, { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_call_summary cs; - int r, err, num_params = 5; + int r, err, callee, arg_slot_cnt = 5; - if (bpf_pseudo_call(insn)) - return 0; + if (bpf_pseudo_call(insn)) { + /* + * analyze_subprog() handles static callees. For global callees, + * derive effects from argument types so may_write marks remain + * conservative under freplace. + */ + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0 || !bpf_subprog_is_global(env, callee)) + return 0; + arg_slot_cnt = MAX_BPF_FUNC_ARGS; + } else if (bpf_is_callx(insn)) + /* + * The callee is not known statically. Assume that all arg + * slots are passed and let record_arg_access() conservatively + * mark the stack of all frames as read if any of them is + * derived from a frame pointer. + */ + arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS + MAX_STACK_ARG_SLOTS; + else if (bpf_get_call_summary(env, insn, &cs)) + arg_slot_cnt = cs.arg_slot_cnt; - if (bpf_get_call_summary(env, insn, &cs)) - num_params = cs.num_params; - - for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) { + for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } - for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) { + for (r = 0; r < MAX_STACK_ARG_SLOTS && r < arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; r++) { err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r], r + MAX_BPF_FUNC_REG_ARGS, insn_idx); if (err) @@ -1517,7 +1844,7 @@ struct subprog_at_info { static void print_subprog_arg_access(struct bpf_verifier_env *env, int subprog, struct subprog_at_info *info, - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS]) + struct arg_track *at_stack_in, u32 nslots) { struct bpf_insn *insns = env->prog->insnsi; int start = env->subprog_info[subprog].start; @@ -1533,7 +1860,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, bool has_extra = false; u8 cls = BPF_CLASS(insns[idx].code); bool is_ldx_stx_call = cls == BPF_LDX || cls == BPF_STX || - insns[idx].code == (BPF_JMP | BPF_CALL); + (cls == BPF_JMP && BPF_OP(insns[idx].code) == BPF_CALL); verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); @@ -1550,8 +1877,8 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, has_extra = true; } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - if (arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) + if (arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) has_extra = true; } @@ -1581,11 +1908,11 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) { - if (!arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) { + if (!arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) continue; verbose(env, " fp%+d=", -(r + 1) * 8); - verbose_arg_track(env, &at_stack_in[i][r]); + verbose_arg_track(env, &at_stack_in[(size_t)i * nslots + r]); } } @@ -1595,6 +1922,17 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } } +static void record_stack_ptrs(struct bpf_verifier_env *env, int idx, struct arg_track *at_in) +{ + u16 r, mask = 0; + + for (r = 0; r < MAX_BPF_REG; r++) + if (arg_is_fp(&at_in[r])) + mask |= BIT(r); + + env->insn_aux_data[idx].stack_ptrs |= mask; +} + /* * Compute arg tracking dataflow for a single subprog. * Runs forward fixed-point with arg_track_xfer(), then records @@ -1618,9 +1956,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, int end = env->subprog_info[subprog + 1].start; int po_end = env->subprog_info[subprog + 1].postorder_start; int len = end - start; + u32 nslots = min(subprog_spill_slots(env, start, end), spill_slots_affordable(len)); struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL; struct arg_track at_out[MAX_AT_TRACK_REGS]; - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL; + struct arg_track *at_stack_in = NULL; struct arg_track *at_stack_out = NULL; struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS]; struct arg_track unvisited = { .frame = ARG_UNVISITED }; @@ -1632,19 +1971,19 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (!at_in) goto err_free; - at_stack_in = kvmalloc_objs(*at_stack_in, len, GFP_KERNEL_ACCOUNT); + at_stack_in = kvmalloc_objs(*at_stack_in, (size_t)len * nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_in) goto err_free; - at_stack_out = kvmalloc_objs(*at_stack_out, MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); + at_stack_out = kvmalloc_objs(*at_stack_out, nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_out) goto err_free; for (i = 0; i < len; i++) { for (r = 0; r < MAX_AT_TRACK_REGS; r++) at_in[i][r] = unvisited; - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[i][r] = unvisited; + for (r = 0; r < nslots; r++) + at_stack_in[(size_t)i * nslots + r] = unvisited; } for (r = 0; r < MAX_AT_TRACK_REGS; r++) @@ -1660,8 +1999,8 @@ static int compute_subprog_args(struct bpf_verifier_env *env, } /* Entry: all stack slots are ARG_NONE */ - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[0][r] = none; + for (r = 0; r < nslots; r++) + at_stack_in[r] = none; /* Entry: incoming stack args from caller, or ARG_NONE for main */ for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) @@ -1683,11 +2022,12 @@ static int compute_subprog_args(struct bpf_verifier_env *env, continue; memcpy(at_out, at_in[i], sizeof(at_out)); - memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out)); + memcpy(at_stack_out, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*at_stack_out)); - arg_track_xfer(env, insn, idx, at_out, at_stack_out, + arg_track_xfer(env, insn, idx, at_out, at_stack_out, nslots, at_stack_arg_entry, instance, callsites); - arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out); + arg_track_log(env, insn, idx, at_in[i], &at_stack_in[(size_t)i * nslots], at_out, + at_stack_out, nslots); /* Propagate to successors within this subprogram */ succ = bpf_insn_successors(env, idx); @@ -1704,9 +2044,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, changed |= arg_track_join(env, idx, target, r, &at_in[ti][r], at_out[r]); - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) + for (r = 0; r < nslots; r++) changed |= arg_track_join(env, idx, target, -r - 1, - &at_stack_in[ti][r], at_stack_out[r]); + &at_stack_in[(size_t)ti * nslots + r], + at_stack_out[r]); } } if (changed) @@ -1722,30 +2063,33 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (err) goto err_free; - if (insn->code == (BPF_JMP | BPF_CALL)) { + if (BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) { err = record_call_access(env, instance, at_in[i], idx); if (err) goto err_free; } if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) { + struct spill_snapshot *snap; + kvfree(env->callsite_at_stack[idx]); - env->callsite_at_stack[idx] = - kvmalloc_objs(*env->callsite_at_stack[idx], - MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); - if (!env->callsite_at_stack[idx]) { + env->callsite_at_stack[idx] = kvmalloc_flex(*snap, at, nslots, GFP_KERNEL_ACCOUNT); + snap = env->callsite_at_stack[idx]; + if (!snap) { err = -ENOMEM; goto err_free; } - memcpy(env->callsite_at_stack[idx], - at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS); + snap->slots = nslots; + memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at)); } + + record_stack_ptrs(env, idx, at_in[i]); } info->at_in = at_in; at_in = NULL; info->len = len; - print_subprog_arg_access(env, subprog, info, at_stack_in); + print_subprog_arg_access(env, subprog, info, at_stack_in, nslots); err = 0; err_free: @@ -1770,38 +2114,87 @@ static bool has_fp_args(struct arg_track *args) /* * Merge a freshly analyzed instance into the original. * may_read: union (any pass might read the slot). + * may_write: union (slots written on ANY pass). * must_write: intersection (only slots written on ALL passes are guaranteed). * live_before is recomputed by a subsequent update_instance() on @dst. + * + * The two instances may have settled on different mask widths for the same + * frame, so @dst is widened to cover @src first. A word only @dst has counts + * as zero on the @src side: it unions into may_read as a no-op and intersects + * must_write to empty. */ -static void merge_instances(struct func_instance *dst, struct func_instance *src) +static int merge_instances(struct func_instance *dst, struct func_instance *src) { - int f, i; + struct frame_masks *d, *s; + u32 f, i, w; for (f = 0; f <= dst->depth; f++) { - if (!src->frames[f]) { + s = src->frames[f]; + d = dst->frames[f]; + if (!s) { /* This pass didn't touch frame f — must_write intersects with empty. */ - if (dst->frames[f]) + if (d) for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(d, i, FM_MUST_WRITE), + frame_mask_bits(d)); continue; } - if (!dst->frames[f]) { + if (!d) { /* Previous pass didn't touch frame f — take src, zero must_write. */ - dst->frames[f] = src->frames[f]; + dst->frames[f] = s; src->frames[f] = NULL; for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(s, i, FM_MUST_WRITE), frame_mask_bits(s)); continue; } + d = widen_frame_masks(dst, f, s->words); + if (!d) + return -ENOMEM; for (i = 0; i < dst->insn_cnt; i++) { - dst->frames[f][i].may_read = - spis_or(dst->frames[f][i].may_read, - src->frames[f][i].may_read); - dst->frames[f][i].must_write = - spis_and(dst->frames[f][i].must_write, - src->frames[f][i].must_write); + unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ); + unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE); + unsigned long *dst_may_write = rel_mask(d, i, FM_MAY_WRITE); + unsigned long *src_read = rel_mask(s, i, FM_MAY_READ); + unsigned long *src_write = rel_mask(s, i, FM_MUST_WRITE); + unsigned long *src_may_write = rel_mask(s, i, FM_MAY_WRITE); + + for (w = 0; w < d->words; w++) { + dst_read[w] |= w < s->words ? src_read[w] : 0; + dst_write[w] &= w < s->words ? src_write[w] : 0; + dst_may_write[w] |= w < s->words ? src_may_write[w] : 0; + } } } + return 0; +} + +/* + * Fold a fully analyzed callee instance writes to upper frames as + * may_write marks at callsite in caller's frames. + */ +static int merge_may_write(struct func_instance *caller, struct func_instance *callee) +{ + DECLARE_BITMAP(acc, FRAME_HALF_SPIS); + u32 call_idx = callee->callsite; + struct frame_masks *fm; + u32 f, i, nbits; + int err; + + for (f = 0; f < callee->depth; f++) { + fm = callee->frames[f]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + bitmap_zero(acc, nbits); + for (i = 0; i < callee->insn_cnt; i++) + bitmap_or(acc, acc, rel_mask(fm, i, FM_MAY_WRITE), nbits); + if (bitmap_empty(acc, nbits)) + continue; + err = mark_stack_mask(caller, f, call_idx, FM_MAY_WRITE, acc, fm->words); + if (err) + return err; + } + return 0; } static struct func_instance *fresh_instance(struct func_instance *src) @@ -1898,7 +2291,12 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (bpf_pseudo_call(insn)) { target = idx + insn->imm + 1; callee = bpf_find_subprog(env, target); - if (callee < 0) + /* + * Global subprograms can be freplaced, so conservatively derive + * their stack effects from their BTF signatures instead of + * analyzing them in the context of their callers. + */ + if (callee < 0 || bpf_subprog_is_global(env, callee)) continue; /* Build entry args: R1-R5 and stack args from at_in at call site */ @@ -1916,7 +2314,10 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (info[subprog].at_in[j][caller_reg].frame == ARG_NONE) continue; for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, idx, SPIS_ALL); + err = mark_stack_read_all(env, instance, f, idx); + if (err) + goto out_free; + err = mark_stack_may_write_all(env, instance, f, idx); if (err) goto out_free; } @@ -1955,23 +2356,35 @@ static int analyze_subprog(struct bpf_verifier_env *env, /* Pull callee's entry liveness back to caller's callsite */ { u32 callee_start = callee_instance->subprog_start; - struct per_frame_masks *entry; + struct frame_masks *callee_fm; for (int f = 0; f < callee_instance->depth; f++) { - entry = get_frame_masks(callee_instance, f, callee_start); - if (!entry) + callee_fm = callee_instance->frames[f]; + if (!callee_fm) continue; - err = mark_stack_read(instance, f, idx, entry->live_before); + err = mark_stack_read_mask(instance, f, idx, + rel_mask(callee_fm, + relative_idx(callee_instance, + callee_start), + FM_LIVE_BEFORE), + callee_fm->words); if (err) goto out_free; } } + + /* Summarize callee's writes to ancestor frames onto the callsite */ + err = merge_may_write(instance, callee_instance); + if (err) + goto out_free; } if (prev_instance) { - merge_instances(prev_instance, instance); + err = merge_instances(prev_instance, instance); free_instance(instance); instance = prev_instance; + if (err) + return err; } update_instance(env, instance); return 0; @@ -2026,6 +2439,8 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env) goto out; } + compute_may_write_masks(env); + if (env->log.level & BPF_LOG_LEVEL2) err = print_instances(env); @@ -2057,10 +2472,17 @@ static inline u32 mask_widen(u32 m) { return m | (m << 16); } static inline u16 mask_lo(u32 m) { return (u16)m; } static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } +/* Index of the subprogram called by pseudo call @insn */ +static int callee_subprog(struct bpf_verifier_env *env, const struct bpf_insn *insn) +{ + return bpf_find_subprog(env, insn - env->prog->insnsi + insn->imm + 1); +} + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2494,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); + int subprog, reg_args; u32 def = 0; u32 use = U32_MAX; @@ -2191,15 +2615,28 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); - if (bpf_get_call_summary(env, insn, &cs)) - use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + if (bpf_get_call_summary(env, insn, &cs)) { + use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); + } else if (bpf_pseudo_call(insn)) { + /* a static callee's reads are added in bpf_compute_live_registers() */ + subprog = callee_subprog(env, insn); + if (bpf_subprog_is_global(env, subprog)) { + reg_args = min_t(u8, env->subprog_info[subprog].arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + use = GENMASK(reg_args, 1); + } else { + use = 0; + } + } def = mask_widen(def); use = mask_widen(use); + /* callx reads the address of the callee from dst_reg */ + if (bpf_is_callx(insn)) + use |= dst; break; default: def = 0; @@ -2227,9 +2664,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; - u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2692,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); @@ -2273,11 +2715,21 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_iarray *succ; u32 new_out = 0; u32 new_in = 0; + u32 use; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) new_out |= state[succ->items[s]].in; - new_in = (new_out & ~live->def) | live->use; + use = live->use; + /* + * For a call to a static subprogram include the callee's + * entry 'in' as computed so far. + */ + if (bpf_pseudo_call(&insns[insn_idx]) && + !bpf_subprog_is_global(env, callee_subprog(env, &insns[insn_idx]))) + use |= state[insn_idx + insns[insn_idx].imm + 1].in & + mask_widen(GENMASK(BPF_REG_5, BPF_REG_1)); + new_in = (new_out & ~live->def) | use; if (new_out != live->out || new_in != live->in) { live->in = new_in; live->out = new_out; @@ -2301,31 +2753,6 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); } - if (env->log.level & BPF_LOG_LEVEL2) { - verbose(env, "Live regs before insn:\n"); - for (i = 0; i < insn_cnt; ++i) { - if (env->insn_aux_data[i].scc) - verbose(env, "%3d ", env->insn_aux_data[i].scc); - else - verbose(env, " "); - verbose(env, "%3d: ", i); - for (j = BPF_REG_0; j < BPF_REG_10; ++j) - if (insn_aux[i].live_regs_before & BIT(j)) - verbose(env, "%d", j); - else - verbose(env, "."); - verbose(env, " "); - pos = env->log.end_pos; - bpf_verbose_insn(env, &insns[i]); - insn_pos = env->log.end_pos; - if (insn_aux[i].zext_dst) - verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); - verbose(env, "\n"); - if (bpf_is_ldimm64(&insns[i])) - i++; - } - } - out: kvfree(state); return err;
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index fb032df..5432c0f 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c
@@ -527,7 +527,7 @@ static void verbose_unum(struct bpf_verifier_env *env, u64 num) verbose(env, "%#llx", num); } -static void verbose_snum(struct bpf_verifier_env *env, s64 num) +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) { if (is_snum_decimal(num)) verbose(env, "%lld", num); @@ -609,7 +609,7 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } if (m1->name[0] == 's') - verbose_snum(env, m1->val); + bpf_verbose_snum(env, m1->val); else verbose_unum(env, m1->val); } @@ -632,7 +632,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); return; } @@ -643,7 +643,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); if (tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value + reg->delta); + bpf_verbose_snum(env, reg->var_off.value + reg->delta); return; } } @@ -667,7 +667,7 @@ static void print_reg_state(struct bpf_verifier_env *env, } if (t != SCALAR_VALUE && reg->delta) { verbose_a("off="); - verbose_snum(env, reg->delta); + bpf_verbose_snum(env, reg->delta); } if (type_is_pkt_pointer(t)) { verbose_a("r="); @@ -683,7 +683,7 @@ static void print_reg_state(struct bpf_verifier_env *env, /* a pointer register with fixed offset */ if (reg->var_off.value) { verbose_a("imm="); - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); } } else { print_scalar_ranges(env, reg, &sep); @@ -693,6 +693,8 @@ static void print_reg_state(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose_a("var_off=%s", tn_buf); } + if (reg->base != 0 || reg->step != 1) + verbose_a("step=%d+%d", reg->base, reg->step); } verbose(env, ")"); } @@ -716,7 +718,8 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, "="); print_reg_state(env, state, reg); } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *slot = bpf_stack_slot(state, i); char types_buf[BPF_REG_SIZE + 1]; const char *sep = ""; bool valid = false; @@ -727,7 +730,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie continue; for (j = 0; j < BPF_REG_SIZE; j++) { - slot_type = state->stack[i].slot_type[j]; + slot_type = slot->slot_type[j]; if (slot_type != STACK_INVALID && slot_type != STACK_POISON) valid = true; types_buf[j] = slot_type_char[slot_type]; @@ -736,12 +739,12 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie if (!valid) continue; - reg = &state->stack[i].spilled_ptr; - switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { + reg = &slot->spilled_ptr; + switch (slot->slot_type[BPF_REG_SIZE - 1]) { case STACK_SPILL: /* print MISC/ZERO/INVALID slots above subreg spill */ for (j = 0; j < BPF_REG_SIZE; j++) - if (state->stack[i].slot_type[j] == STACK_SPILL) + if (slot->slot_type[j] == STACK_SPILL) break; types_buf[j] = '\0'; @@ -751,7 +754,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie case STACK_DYNPTR: /* skip to main dynptr slot */ i += BPF_DYNPTR_NR_SLOTS - 1; - reg = &state->stack[i].spilled_ptr; + reg = &bpf_stack_slot(state, i)->spilled_ptr; verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); @@ -789,6 +792,11 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, " cb"); if (state->in_async_callback_fn) verbose(env, " async_cb"); + if (state->loop_stack_cnt) { + verbose(env, " loop_stack="); + for (i = 0; i < state->loop_stack_cnt; i++) + verbose(env, "%s%d", i ? "," : "", state->loop_stack[i].loop_id); + } verbose(env, "\n"); if (!print_all) mark_verifier_state_clean(env);
diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c new file mode 100644 index 0000000..2568079 --- /dev/null +++ b/kernel/bpf/loops.c
@@ -0,0 +1,616 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/slab.h> +#include <linux/sched/signal.h> +#include <linux/bpf_verifier.h> + +static bool is_cfg_jump(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + switch (class) { + case BPF_JMP: + case BPF_JMP32: + return opcode != BPF_CALL; + default: + return false; + } +} + +static struct bpf_iarray **compute_predecessors(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_iarray *succ, *preds, **result; + struct bpf_prog *prog = env->prog; + u32 *num_preds, i, s, len = prog->len; + struct bpf_insn *insn; + size_t sz; + void *tmp; + + num_preds = kvcalloc(prog->len, sizeof(u32), GFP_KERNEL_ACCOUNT); + if (!num_preds) + return NULL; + + /* + * 'result' layout: + * - array of pointers (struct bpf_iarray *)[len] + * - struct bpf_iarray one after another + */ + sz = size_mul(len, sizeof(*result) + sizeof(**result)); + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + sz = size_add(sz, array_size(succ->cnt, sizeof(*succ->items))); + iarray_for_each(s, succ) { + num_preds[s]++; + } + if (bpf_is_ldimm64(insn)) + i++; + } + + result = kvzalloc(sz, GFP_KERNEL_ACCOUNT); + if (!result) { + kvfree(num_preds); + return NULL; + } + + tmp = (void *)&result[len]; + for (i = 0; i < len; i++) { + result[i] = tmp; + tmp += struct_size(result[i], items, num_preds[i]); + } + + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + preds = result[s]; + preds->items[preds->cnt++] = i; + } + if (succ->cnt > 1 || + (succ->cnt == 1 && num_preds[succ->items[0]] > 1) || + is_cfg_jump(insn)) + aux[i].bb_end = true; + if (bpf_is_ldimm64(insn)) + i++; + } + + kvfree(num_preds); + return result; +} + +static int idoms_intersect(struct bpf_verifier_env *env, int a, int b) +{ + int *postorder_nums = env->cfg.postorder_nums; + int *idoms = env->idoms; + + while (a != b) { + while (postorder_nums[a] < postorder_nums[b]) { + a = idoms[a]; + } + while (postorder_nums[b] < postorder_nums[a]) { + b = idoms[b]; + } + } + return a; +} + +/* See "A Simple, Fast Dominance Algorithm" by Cooper et al. for details. */ +static int compute_subprog_idoms(struct bpf_verifier_env *env, struct bpf_iarray **preds, + int subprog_idx) +{ + struct bpf_subprog_info *subprog = &env->subprog_info[subprog_idx]; + int start = subprog->start; + int po_first = subprog->postorder_start; + int po_last = (subprog + 1)->postorder_start - 1; + int *idoms = env->idoms; + int po_num, pred; + u32 work = 0; + bool changed; + + idoms[start] = start; + changed = true; + do { + changed = false; + /* + * Iterate in reverse postorder, skipping the subprog entry: + * insn_postorder[po_last] is the subprogram entry. + */ + for (po_num = po_last - 1; po_num >= po_first; po_num--) { + int idx = env->cfg.insn_postorder[po_num]; + int new_idom = -1; + + iarray_for_each(pred, preds[idx]) { + if (++work % 1024 == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + if (idoms[pred] == -1) + continue; + if (new_idom == -1) + new_idom = pred; + else + new_idom = idoms_intersect(env, pred, new_idom); + } + if (new_idom != -1 && idoms[idx] != new_idom) { + idoms[idx] = new_idom; + changed = true; + } + } + } while (changed); + idoms[start] = -1; + return 0; +} + +int bpf_compute_idoms(struct bpf_verifier_env *env) +{ + struct bpf_iarray **preds; + u32 len = env->prog->len; + int *idoms, i, err = 0; + + preds = compute_predecessors(env); + if (!preds) + return -ENOMEM; + + idoms = kvcalloc(len, sizeof(*idoms), GFP_KERNEL_ACCOUNT); + if (!idoms) { + kvfree(preds); + return -ENOMEM; + } + + env->idoms = idoms; + for (i = 0; i < len; i++) + idoms[i] = -1; + + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_subprog_idoms(env, preds, i); + if (err) + break; + } + + kvfree(preds); + return err; +} + +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + +struct loops_dfs { + struct dfs_state *state; + int *dfs_pos; + int *stack; +}; + +static void mark_irreducible(struct bpf_verifier_env *env, int h) +{ + env->insn_aux_data[h].loop->irreducible = true; +} + +static void add_backedge(struct bpf_verifier_env *env, int from, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[h].loop; + int cnt = loop->backedges_cnt; + + if (cnt == MAX_BACKEDGES) { + loop->backedges_overflow = true; + return; + } + loop->backedges[cnt].from = from; + loop->backedges[cnt].latches_cnt = 0; + loop->backedges_cnt++; +} + +static int add_exit(struct bpf_loop *loop, int from, int to) +{ + if (loop->exits_overflow) + return 0; + if (loop->exits_cnt == MAX_LOOP_EXITS) { + loop->exits_overflow = true; + return 0; + } + if (!loop->exits) { + loop->exits = kvcalloc(MAX_LOOP_EXITS, sizeof(*loop->exits), GFP_KERNEL_ACCOUNT); + if (!loop->exits) + return -ENOMEM; + } + loop->exits[loop->exits_cnt] = (struct bpf_loop_exit) { + .from = from, + .to = to, + }; + loop->exits_cnt++; + return 0; +} + +static int mark_as_header(struct bpf_verifier_env *env, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + if (!aux[h].loop) { + aux[h].loop = kvzalloc_obj(struct bpf_loop, GFP_KERNEL_ACCOUNT); + if (!aux[h].loop) + return -ENOMEM; + } + return 0; +} + +static int assign_header(struct bpf_verifier_env *env, struct loops_dfs *dfs, int n, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int *dfs_pos = dfs->dfs_pos; + int err, nh; + + err = mark_as_header(env, h); + if (err) + return err; + + /* Don't encode self-loops, otherwise can't reflect loops nesting structure. */ + if (n == h) + return 0; + + /* Make sure that loop headers up the chain are sorted by dfs_pos. */ + while (aux[n].loop_header != -1) { + nh = aux[n].loop_header; + if (nh == h) + return 0; + if (dfs_pos[nh] < dfs_pos[h]) { + aux[n].loop_header = h; + n = h; + h = nh; + } else { + n = nh; + } + } + aux[n].loop_header = h; + return 0; +} + +static bool is_cond_jmp_insn(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + if (class != BPF_JMP && class != BPF_JMP32) + return false; + + switch (opcode) { + case BPF_JEQ: + case BPF_JGE: + case BPF_JGT: + case BPF_JLE: + case BPF_JLT: + case BPF_JNE: + case BPF_JSET: + case BPF_JSGE: + case BPF_JSGT: + case BPF_JSLE: + case BPF_JSLT: + return true; + default: + return false; + } +} + +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + return aux[idx].loop ? idx : aux[idx].loop_header; +} + +/* + * Collect conditional jumps within n's loop that dominate 'n' and have + * a successor outside the loop, nearest first. + */ +static void find_dominating_conditions(struct bpf_verifier_env *env, int n, + struct bpf_backedge *backedge) +{ + struct bpf_insn *insns = env->prog->insnsi; + bool t_exits_loop, f_exits_loop; + int n_loop, t_loop, f_loop; + int *idoms = env->idoms; + + n_loop = bpf_loop_at_index(env, n); + if (idoms_intersect(env, n, n_loop) != n_loop) + return; + while (n >= 0) { + if (is_cond_jmp_insn(&insns[n]) && bpf_loop_at_index(env, n) == n_loop) { + t_loop = bpf_loop_at_index(env, n + insns[n].off + 1); + f_loop = bpf_loop_at_index(env, n + 1); + t_exits_loop = t_loop != n_loop && !bpf_is_nested_loop(env, t_loop, n_loop); + f_exits_loop = f_loop != n_loop && !bpf_is_nested_loop(env, f_loop, n_loop); + if (t_exits_loop || f_exits_loop) { + if (backedge->latches_cnt == MAX_LATCHES) + return; + backedge->latches[backedge->latches_cnt++] = n; + } + } + if (n == n_loop) + break; + n = idoms[n]; + } +} + +/* + * As described in "A New Algorithm for Identifying Loops in Decompilation" by Wei et al, + * adapted to be non-recursive. + */ +static int compute_loops_in_subprog(struct bpf_verifier_env *env, struct loops_dfs *dfs, + int subprog_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct dfs_state *state = dfs->state; + int start = env->subprog_info[subprog_idx].start; + int *dfs_pos = dfs->dfs_pos; + int *stack = dfs->stack; + int s, h, err, cur, stack_sz; + struct bpf_iarray *succ; + u32 i; + + stack[0] = start; + state[start].traversed = true; + state[start].next_succ = 0; + dfs_pos[start] = 1; + stack_sz = 1; + i = 0; + do { + /* + * The algorithm should be very fast in practice, + * guard against pathological inputs, just in case. + */ + if ((++i % 1024) == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + dfs_pos[cur] = 0; + stack_sz--; + continue; + } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + /* Case A: start -> ... -> cur -> s [unexplored] */ + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + dfs_pos[s] = stack_sz + 1; + stack_sz++; + continue; + } + /* 's' is fully explored at this point */ + if (dfs_pos[s]) { + /* + * start -> ... -> s -> cur --. + * ^ | + * '----------' + * Case B: 's' is in the current DFS path. + */ + err = assign_header(env, dfs, cur, s); + if (err) + return err; + add_backedge(env, cur, s); + } else if (aux[s].loop_header == -1) { + /* + * start -> ... -> ... -> s -> ... -> end + * | ^ + * '---> cur ---' + * Case C: 's' is explored, not in the current DFS path, + * and not a part of any loop. + */ + } else if (dfs_pos[aux[s].loop_header]) { + /* + * .----------------------. + * v | + * start -> ... -> h -> ... -> ... -> s --' + * | ^ + * '---> cur ---' + * Case D: 's' is explored, not in current DFS path, + * but its innermost loop header is. + */ + err = assign_header(env, dfs, cur, aux[s].loop_header); + if (err) + return err; + } else { + /* + * case E: 's' is explored, not in current DFS path, + * its innermost loop header is not in current DFS path, + * hence 's' is another entry into the same loop. + */ + h = aux[s].loop_header; + mark_irreducible(env, h); + while (aux[h].loop_header != -1) { + h = aux[h].loop_header; + if (dfs_pos[h]) { + err = assign_header(env, dfs, cur, h); + if (err) + return err; + break; + } + mark_irreducible(env, h); + } + } + state[cur].next_succ++; + } while (stack_sz); + + return 0; +} + +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int idx; + + for (idx = inner_header; idx >= 0; idx = aux[idx].loop_header) + if (aux[idx].loop_header == outer_header) + return true; + + return false; +} + +int bpf_compute_loops(struct bpf_verifier_env *env) +{ + int i, j, k, s, t, iloop, sloop, err = 0, len = env->prog->len; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct bpf_backedge *backedge; + struct loops_dfs dfs = {}; + struct bpf_iarray *succ; + struct bpf_loop *loop; + + dfs.dfs_pos = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + dfs.state = kvcalloc(len, sizeof(struct dfs_state), GFP_KERNEL_ACCOUNT); + dfs.stack = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + if (!dfs.dfs_pos || !dfs.state || !dfs.stack) { + err = -ENOMEM; + goto out; + } + for (i = 0; i < len; i++) + aux[i].loop_header = -1; + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_loops_in_subprog(env, &dfs, i); + if (err) + goto out; + } + /* find latches */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + /* + * In theory, the backedge->from and it's latch can reside in + * an inner loop of `i`, e.g.: + * + * 1: r7 += 1; + * 2: r6 += 1; + * if r7 == 2 goto 1b; + * if r6 < 2 goto 2b; + * + * For now, let's assume that latches are unknown for such cases. + * (GCC/LLVM handle this by inserting artificial cfg nodes). + */ + if (bpf_loop_at_index(env, backedge->from) != i) + continue; + find_dominating_conditions(env, backedge->from, backedge); + } + } + /* find exits */ + for (i = 0; i < len; i++) { + iloop = aux[i].loop ? i : aux[i].loop_header; + if (iloop < 0) + continue; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + /* + * Nothing left to record once the innermost loop of 'i' + * overflowed: the walk below would stop at it right away. + */ + if (aux[iloop].loop->exits_overflow) + break; + sloop = aux[s].loop ? s : aux[s].loop_header; + if (iloop == sloop) + continue; + if (bpf_is_nested_loop(env, sloop, iloop)) + continue; + /* + * At this point 'sloop' is either -1, an outer loop, + * or a loop in another branch of the loop hierarchy. + */ + for (t = iloop; t >= 0 && t != sloop; t = aux[t].loop_header) { + /* + * Account for the following configuration: + * + * tloop { + * iloop { + * ... i: goto s; + * } + * sloop { + * s: + * ... + * } + * } + */ + if (bpf_is_nested_loop(env, sloop, t)) + break; + /* + * Record edges i -> s as exits from tloop when: + * + * sloop { + * tloop { + * iloop { + * ... i: goto s; + * } + * } + * s: ... + * } + */ + /* Enclosing loops inherit the overflow, see below */ + if (aux[t].loop->exits_overflow) + break; + err = add_exit(aux[t].loop, i, s); + if (err) + goto out; + } + } + if (bpf_is_ldimm64(env->prog->insnsi + i)) + i++; + } + /* A nested loop with a truncated exit list can't be abstracted by SCEV */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop || !loop->exits_overflow) + continue; + for (t = aux[i].loop_header; t >= 0; t = aux[t].loop_header) + aux[t].loop->exits_overflow = true; + } + + if (env->log.level & BPF_LOG_LEVEL2) { + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + bpf_log(log, "loop at %d", i); + if (aux[i].loop_header >= 0) + bpf_log(log, ", nested in %d", aux[i].loop_header); + if (loop->irreducible) + bpf_log(log, ", irreducible"); + if (loop->exits_overflow) + bpf_log(log, ", too many exits"); + bpf_log(log, "\n"); + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + bpf_log(log, " backedge from %d", backedge->from); + for (k = 0; k < backedge->latches_cnt; k++) { + if (k == 0) + bpf_log(log, ", latch at "); + if (k == 1) + bpf_log(log, ", dominating exits at "); + if (k > 1) + bpf_log(log, ", "); + bpf_log(log, "%d", backedge->latches[k]); + } + bpf_log(log, "\n"); + } + for (j = 0; j < loop->exits_cnt; j++) + bpf_log(log, " exit from %d to %d\n", + loop->exits[j].from, loop->exits[j].to); + } + } + +out: + kvfree(dfs.dfs_pos); + kvfree(dfs.stack); + kvfree(dfs.state); + return err; +}
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index d2cbab4..0fa3f2a 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c
@@ -25,6 +25,10 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) if (!inner_map->ops->map_meta_equal) return ERR_PTR(-ENOTSUPP); + /* An inner map has no used_maps reference to hold it under a queued callback. */ + if (btf_record_has_field(inner_map->record, BPF_RCU_HEAD)) + return ERR_PTR(-EOPNOTSUPP); + inner_map_meta_size = sizeof(*inner_map_meta); /* In some cases verifier needs to access beyond just base map. */ if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) @@ -106,10 +110,13 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return ERR_PTR(-ENOTSUPP); inner_map_meta = map->inner_map_meta; - if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) + if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) { bpf_map_inc(inner_map); - else + /* Its users are those of the outer maps from now on */ + WRITE_ONCE(inner_map->was_inner_map, true); + } else { inner_map = ERR_PTR(-EINVAL); + } return inner_map; }
diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index c19b360b..a58c148 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c
@@ -97,13 +97,57 @@ static struct bpf_iter_reg bpf_map_reg_info = { .seq_info = &bpf_map_seq_info, }; +/* + * For an iterator that hands the live map value to the program as a writable + * buffer: the verifier knows nothing of the map, so the special fields of the + * value have to lie outside of what the program may write. + */ +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map) +{ + const struct btf_record *rec = map->record; + const struct btf_field *field; + u32 i; + + if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write) + return 0; + + for (i = 0; i < rec->cnt; i++) { + field = &rec->fields[i]; + if (field->offset < prog->aux->max_rdwr_write && + field->offset + field->size > prog->aux->min_rdwr_write) + return -EACCES; + } + + return 0; +} + +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + const struct bpf_map *map = aux->map; + u32 value_size = map->value_size; + bool is_percpu; + + is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; + if (is_percpu) + value_size = round_up(value_size, 8) * num_possible_cpus(); + + if (prog->aux->max_rdonly_access > map->key_size || + prog->aux->max_rdwr_access > value_size) + return -EACCES; + + /* the program gets a copy of a per-cpu value */ + return is_percpu ? 0 : bpf_iter_map_check_value_write(prog, map); +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) { - u32 key_acc_size, value_acc_size, key_size, value_size; struct bpf_map *map; - bool is_percpu = false; int err = -EINVAL; if (!linfo->map.map_fd) @@ -117,30 +161,26 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, goto put_map; } - if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) - is_percpu = true; - else if (map->map_type != BPF_MAP_TYPE_HASH && - map->map_type != BPF_MAP_TYPE_LRU_HASH && - map->map_type != BPF_MAP_TYPE_ARRAY && - map->map_type != BPF_MAP_TYPE_RHASH) - goto put_map; - - key_acc_size = prog->aux->max_rdonly_access; - value_acc_size = prog->aux->max_rdwr_access; - key_size = map->key_size; - if (!is_percpu) - value_size = map->value_size; - else - value_size = round_up(map->value_size, 8) * num_possible_cpus(); - - if (key_acc_size > key_size || value_acc_size > value_size) { - err = -EACCES; + /* The value ctx arg is writable and aliases the live element. */ + if (btf_record_has_field(map->record, BPF_RCU_HEAD)) { + err = -EOPNOTSUPP; goto put_map; } + if (map->map_type != BPF_MAP_TYPE_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && + map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_LRU_HASH && + map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_RHASH) + goto put_map; + aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -173,6 +213,7 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { .target = "bpf_map_elem", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2,
diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index d855399..f7bfff1 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c
@@ -96,6 +96,22 @@ static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, return err; } +/* + * @prog takes the place of a copy that the caller is about to free. Nothing + * keeps the netdev while a program is loaded: once the lock is dropped, no + * unregistration is left that has got the old copy from offload->prog. + */ +void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ + if (!bpf_prog_is_dev_bound(prog->aux)) + return; + + down_read(&bpf_devs_lock); + if (prog->aux->offload) + prog->aux->offload->prog = prog; + up_read(&bpf_devs_lock); +} + static void __bpf_prog_offload_destroy(struct bpf_prog *prog) { struct bpf_prog_offload *offload = prog->aux->offload; @@ -250,7 +266,14 @@ int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr) prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY); down_write(&bpf_devs_lock); - err = __bpf_prog_dev_bound_init(prog, netdev); + /* + * Behind bpf_dev_bound_netdev_unregister() nobody would take the + * program off the device, which goes once the reference is dropped. + */ + if (READ_ONCE(netdev->reg_state) == NETREG_REGISTERED) + err = __bpf_prog_dev_bound_init(prog, netdev); + else + err = -EINVAL; up_write(&bpf_devs_lock); out:
diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 6e8b18c..1cee2ba 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c
@@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr) */ return -E2BIG; + if (attr->max_entries >= U32_MAX / attr->value_size) + return -E2BIG; + return 0; }
diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 2f28886..2460e78 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c
@@ -143,16 +143,22 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len) if (rn->rn_start < start && rn->rn_last > last) { u32 old_last = rn->rn_last; + /* + * Pre-allocate the right-half node before modifying + * the tree. If allocation fails we return -ENOMEM + * without altering the range tree. + */ + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + /* Overlaps with the entire clearing range */ range_it_remove(rn, rt); rn->rn_last = start - 1; range_it_insert(rn, rt); - /* Add a range */ - new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, - NUMA_NO_NODE); - if (!new_rn) - return -ENOMEM; + /* Add right-half range */ new_rn->rn_start = last + 1; new_rn->rn_last = old_last; range_it_insert(new_rn, rt); @@ -193,6 +199,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len) int range_tree_set(struct range_tree *rt, u32 start, u32 len) { u32 last = start + len - 1; + struct range_node *new_rn = NULL; struct range_node *right; struct range_node *left; int err; @@ -202,20 +209,19 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) if (left && left->rn_start <= start && left->rn_last >= last) return 0; + left = range_it_iter_first(rt, start - 1, start - 1); + right = range_it_iter_first(rt, last + 1, last + 1); + if (!left && !right) { + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + } + /* Clear out everything in the range we want to set. */ err = range_tree_clear(rt, start, len); if (err) - return err; - - /* Do we have a left-adjacent range ? */ - left = range_it_iter_first(rt, start - 1, start - 1); - if (left && left->rn_last + 1 != start) - return -EFAULT; - - /* Do we have a right-adjacent range ? */ - right = range_it_iter_first(rt, last + 1, last + 1); - if (right && right->rn_start != last + 1) - return -EFAULT; + goto out_free_new; if (left && right) { /* Combine left and right adjacent ranges */ @@ -235,14 +241,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start; range_it_insert(right, rt); } else { - left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!left) - return -ENOMEM; - left->rn_start = start; - left->rn_last = last; - range_it_insert(left, rt); + /* No adjacent ranges; use the pre-allocated node */ + new_rn->rn_start = start; + new_rn->rn_last = last; + range_it_insert(new_rn, rt); } return 0; + +out_free_new: + kfree_nolock(new_rn); + return err; } void range_tree_destroy(struct range_tree *rt)
diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 3f1013d..447863e 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c
@@ -722,7 +722,7 @@ BPF_CALL_2(bpf_ringbuf_submit_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags) const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto = { .func = bpf_ringbuf_submit_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; @@ -741,7 +741,7 @@ BPF_CALL_2(bpf_ringbuf_discard_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = { .func = bpf_ringbuf_discard_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, };
diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c new file mode 100644 index 0000000..fea7777 --- /dev/null +++ b/kernel/bpf/scev.c
@@ -0,0 +1,2916 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include "linux/cnum.h" +#include <linux/bpf_verifier.h> +#include <linux/jhash.h> +#include <linux/log2.h> +#include <linux/math64.h> +#include <linux/min_heap.h> +#include <linux/bug.h> +#include <linux/tnum.h> +#include <linux/overflow.h> + +#define REGS_NUM BPF_SCEV_REGS_NUM +#define UNKNOWN_EXPR_ID 0 +#define OPAQUE_EXPR_ID 1 + +/* + * BPF instructions use 'code', 'src_reg', 'off' and 'imm' fields for instruction encoding. + * For scalar evolution purpose we want to reuse most of the opcode definitions, + * but also add a few custom operations (REG and IMM). + * Use the enum below to uniformly represent all operations relevant for SCEV. + */ +enum expr_op { + /* leave range 0..255 for standard bpf opcodes */ + UNKNOWN = 256, /* start custom opcodes from the second byte */ + REG, + IMM, + SDIV, SMOD, + SEXT8, SEXT16, SEXT32, + ZEXT8, ZEXT16, ZEXT32, + BSWAP16, BSWAP32, BSWAP64, + /* during the loop body execution the value can be either of param[0] or param[1] */ + ANY, + /* some value that verifier is not going to track precisely */ + OPAQUE, + /* '*(u8/16/32 *)(r10 + X) = Y' writes define slot contents only partially */ + SPILL8, SPILL16, SPILL32, + /* + * SCEV expression corresponding to linear equation 'param[0] + param[1] * k', + * where k is a loop iteration number. Loop here refers to innermost loop + * containing instruction associated with this expression, as returned by + * bpf_loop_at_index(). + */ + LINEAR_SCEV, +}; + +struct expr { + u32 op; + union { + u32 params[2]; + s64 imm; + }; +}; + +struct expr_bucket { + u32 cnt; + u32 cap; + u32 ids[]; +}; + +struct env { + bool empty; + u32 reg2expr[REGS_NUM]; + u32 reg2scev[REGS_NUM]; +}; + +struct insn_envs { + u32 cnt; + struct { + int loop_header; + struct env *env; + } entries[]; +}; + +#define EXPR_STACK_DEPTH 8 + +struct expr_stack_elt { + u32 id:28; + u32 pre:1; + u32 next_param:2; +}; + +DEFINE_MIN_HEAP(int, scev_worklist); + +struct scev { + /* Read-only representatives for stack slots without a saved register. */ + struct bpf_reg_state zero_reg; + struct bpf_reg_state unknown_reg; + /* + * Expressions are identified by id, exprs_ht ensures that + * each expression exists as a unique instance. + * This allows for fast equivalence check: id1 === id2. + */ + struct expr_bucket **exprs_ht; + struct scev_worklist worklist; + struct insn_envs **envs; + struct expr *exprs; + /* + * Loops are analyzed one by one, this array keeps track if a particular + * instruction was visited on a current pass. + */ + u32 *discovered; + u32 exprs_ht_cnt; + int exprs_cnt; + int exprs_cap; + int envs_cnt; + int stack_sz; + struct expr_stack_elt expr_stack[EXPR_STACK_DEPTH]; + u32 ids_buf[EXPR_STACK_DEPTH]; +}; + +static u32 expr_hash(struct expr *e) +{ + return jhash_3words(e->op, e->params[0], e->params[1], 0); +} + +static int expr_eq(struct expr *a, struct expr *b) +{ + return a->op == b->op && a->imm == b->imm; +} + +static int add_expr(struct scev *scev, struct expr e) +{ + struct expr_bucket *bucket; + u32 i, id, hash, new_cap; + void *tmp; + + hash = expr_hash(&e) & (scev->exprs_ht_cnt - 1); + bucket = scev->exprs_ht[hash]; + + if (bucket) { + for (i = 0; i < bucket->cnt; i++) { + id = bucket->ids[i]; + if (expr_eq(&e, &scev->exprs[id])) + return id; + } + } + + if (!bucket || bucket->cap == bucket->cnt) { + new_cap = bucket ? bucket->cap * 2 : 4; + bucket = kvrealloc(bucket, sizeof(*bucket) + sizeof(u32) * new_cap, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!bucket) + return -ENOMEM; + scev->exprs_ht[hash] = bucket; + bucket->cap = new_cap; + } + + if (scev->exprs_cnt == scev->exprs_cap) { + new_cap = scev->exprs_cap ? scev->exprs_cap * 2 : 256; + tmp = kvrealloc(scev->exprs, sizeof(struct expr) * new_cap, GFP_KERNEL_ACCOUNT); + if (!tmp) + return -ENOMEM; + scev->exprs = tmp; + scev->exprs_cap = new_cap; + } + + id = scev->exprs_cnt++; + scev->exprs[id] = e; + bucket->ids[bucket->cnt++] = id; + return id; +} + +static int expr2(struct scev *scev, u32 op, int a, int b) +{ + if (a < 0) + return a; + if (b < 0) + return b; + return add_expr(scev, (struct expr){ .op = op, .params = {a, b} }); +} + +static int expr1(struct scev *scev, u32 op, int a) +{ + if (a < 0) + return a; + return expr2(scev, op, a, 0); +} + +static int expr0(struct scev *scev, u32 op) +{ + return expr2(scev, op, 0, 0); +} + +static int imm_expr(struct scev *scev, s64 value) +{ + return add_expr(scev, (struct expr){ .op = IMM, .imm = value }); +} + +static bool same_exprs(struct scev *scev, int id_a, int id_b) +{ + return id_a == id_b; +} + +static bool is_imm(struct scev *scev, int id, s64 *imm) +{ + if (scev->exprs[id].op != IMM) + return false; + *imm = scev->exprs[id].imm; + return true; +} + +static bool is_op(struct scev *scev, int id, u32 op) +{ + if (scev->exprs[id].op != op) + return false; + return true; +} + +static bool is_unop(struct scev *scev, enum expr_op op, int id, u32 *p0) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + return true; +} + +static bool is_binop(struct scev *scev, enum expr_op op, int id, u32 *p0, u32 *p1) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + *p1 = scev->exprs[id].params[1]; + return true; +} + +static bool is_reg(struct scev *scev, int id, u32 *reg) +{ + return is_unop(scev, REG, id, reg); +} + +static bool is_add(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, BPF_ADD, id, left, right); +} + +static bool is_zext32(struct scev *scev, int id, u32 *left) +{ + return is_unop(scev, ZEXT32, id, left); +} + +static bool is_any(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, ANY, id, left, right); +} + +static bool is_linear(struct scev *scev, int id, u32 *base, u32 *slope) +{ + return is_binop(scev, LINEAR_SCEV, id, base, slope); +} + +static bool is_linear_or_linear32(struct scev *scev, int id, u32 *base, u32 *slope, bool *zext32) +{ + u32 t; + + *zext32 = is_zext32(scev, id, &t); + return is_linear(scev, *zext32 ? t : id, base, slope); +} + +static bool is_opaque(struct scev *scev, int id) +{ + return is_op(scev, id, OPAQUE); +} + +static void log_reg(struct bpf_verifier_env *env, u32 reg) +{ + if (reg < MAX_BPF_REG) + bpf_log(&env->log, "r%d", reg); + else + bpf_log(&env->log, "*fp%d", (MAX_BPF_REG - reg - 1) * 8); +} + +static const char *op_str(u32 op) +{ + switch (op) { + case BPF_ADD: return "+"; + case BPF_SUB: return "-"; + case BPF_MUL: return "*"; + case BPF_DIV: return "/"; + case SDIV: return "s/"; + case BPF_OR: return "|"; + case BPF_AND: return "&"; + case BPF_LSH: return "<<"; + case BPF_RSH: return ">>"; + case BPF_NEG: return "-"; + case BPF_MOD: return "%"; + case SMOD: return "s%"; + case BPF_XOR: return "^"; + case BPF_ARSH: return "s>>"; + case SEXT8: return "sext8"; + case SEXT16: return "sext16"; + case SEXT32: return "sext32"; + case ZEXT8: return "zext8"; + case ZEXT16: return "zext16"; + case ZEXT32: return "zext32"; + case BSWAP16: return "bswap16"; + case BSWAP32: return "bswap32"; + case BSWAP64: return "bswap64"; + case SPILL8: return "spill8"; + case SPILL16: return "spill16"; + case SPILL32: return "spill32"; + case ANY: return "any"; + case LINEAR_SCEV: return "linear"; + } + return NULL; +} + +static u32 op_params_num(u32 op) +{ + switch (op) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case SDIV: + case SMOD: + case ANY: + case LINEAR_SCEV: + return 2; + case BPF_NEG: + case SEXT8: + case SEXT16: + case SEXT32: + case ZEXT8: + case ZEXT16: + case ZEXT32: + case BSWAP16: + case BSWAP32: + case BSWAP64: + case SPILL8: + case SPILL16: + case SPILL32: + return 1; + case REG: + case IMM: + case OPAQUE: + return 0; + default: + return 0; + } +} + +static bool expr_stack_push(struct scev *scev, u32 id) +{ + if (scev->stack_sz >= EXPR_STACK_DEPTH) + return false; + scev->expr_stack[scev->stack_sz].id = id; + scev->expr_stack[scev->stack_sz].pre = true; + scev->expr_stack[scev->stack_sz].next_param = 0; + scev->stack_sz++; + return true; +} + +enum { + PRE = BIT(1), POST = BIT(2), DEPTH_LIMIT = BIT(3) +}; + +static bool expr_next(struct scev *scev, u32 *id, u32 *order) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 num_params; + + if (scev->stack_sz == 0) + return false; + + elt = &scev->expr_stack[scev->stack_sz - 1]; + *id = elt->id; + *order = 0; + expr = &scev->exprs[elt->id]; + num_params = op_params_num(expr->op); + if (elt->pre) { + elt->pre = false; + *order = PRE; + return true; + } + if (elt->next_param == num_params) { + *order = POST; + scev->stack_sz--; + return true; + } + if (scev->stack_sz == EXPR_STACK_DEPTH) { + *order = POST | DEPTH_LIMIT; + scev->stack_sz--; + return true; + } + expr_stack_push(scev, expr->params[elt->next_param]); + elt->next_param++; + return expr_next(scev, id, order); +} + +static void log_expr(struct bpf_verifier_env *env, u32 id) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct expr *expr; + const char *str; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->stack_sz > 1) + bpf_log(log, " "); + expr = &scev->exprs[id]; + switch (expr->op) { + case UNKNOWN: + if (order & PRE) + bpf_log(log, "?"); + break; + case OPAQUE: + if (order & PRE) + bpf_log(log, "_"); + break; + case REG: + if (order & PRE) + log_reg(env, expr->params[0]); + break; + case IMM: + if (order & PRE) + bpf_log(log, "%lld", expr->imm); + break; + default: + if (order & PRE) { + str = op_str(expr->op); + bpf_log(log, "("); + if (str) + bpf_log(log, "%s", str); + else + bpf_log(log, "bad-expr-op %x", expr->op); + } + if (order & DEPTH_LIMIT) + bpf_log(log, "..."); + if (order & POST) + bpf_log(log, ")"); + } + } +} + +static bool reg_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + u16 live_regs = aux[insn_idx].live_regs_before; + + return reg < MAX_BPF_REG + ? (BIT(reg) & live_regs) + : test_bit(reg - MAX_BPF_REG, aux[insn_idx].live_stack_before); +} + +static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + bool printed_some = false; + bool is_self_scev; + bool is_self_reg; + int i, r; + + if (e->empty) { + bpf_log(log, " <empty>\n"); + return; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!reg_alive_at(env, i, insn_idx)) + continue; + is_self_reg = is_reg(scev, e->reg2expr[i], &r) && i == r; + is_self_scev = is_reg(scev, e->reg2scev[i], &r) && i == r; + if (is_self_reg && is_self_scev) + continue; + printed_some = true; + bpf_log(log, " "); + log_reg(env, i); + bpf_log(log, "="); + log_expr(env, e->reg2expr[i]); + if (e->reg2expr[i] != UNKNOWN_EXPR_ID) { + bpf_log(log, " / "); + log_expr(env, e->reg2scev[i]); + } + bpf_log(log, "\n"); + } + if (!printed_some) + bpf_log(log, " <all regs unchanged>\n"); +} +static struct env *find_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 i; + + if (!envs) + return NULL; + for (i = 0; i < envs->cnt; i++) + if (envs->entries[i].loop_header == loop_header) + return envs->entries[i].env; + return NULL; +} + +static struct env *find_header_env(struct scev *scev, int header) +{ + return find_loop_env(scev, header, header); +} + +static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 cnt = envs ? envs->cnt : 0; + struct insn_envs *tmp; + struct env *e; + + e = find_loop_env(scev, loop_header, insn_idx); + if (e) + return e; + + e = kzalloc(sizeof(*e), GFP_KERNEL_ACCOUNT); + if (!e) + return NULL; + + tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), GFP_KERNEL_ACCOUNT); + if (!tmp) { + kfree(e); + return NULL; + } + + e->empty = true; + tmp->entries[cnt].loop_header = loop_header; + tmp->entries[cnt].env = e; + tmp->cnt = cnt + 1; + scev->envs[insn_idx] = tmp; + return e; +} + +static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) +{ + if (id < 0) + return id; + e->reg2expr[reg] = id; + return 0; +} + +static int setup_initial_loop_env(struct bpf_verifier_env *env, struct env *e, int insn_idx) +{ + struct scev *scev = env->scev; + int i, err; + + /* + * Initial assignments are of form rX -> rX are needed as seeds for subsequent + * work queue pop / transfer() calls. Because e->empty is true, the first backedge + * join() will overwrite the them. compute_header_scevs() accounts for register + * states at loop entry. + */ + for (i = 0; i < REGS_NUM; i++) { + err = replace_reg(scev, e, i, expr1(scev, REG, i)); + if (err) + return err; + } + return 0; +} + +static void forget_call_regs(struct env *e) +{ + int i; + + for (i = BPF_REG_0; i <= BPF_REG_5; i++) + e->reg2expr[i] = UNKNOWN_EXPR_ID; +} + +static u32 spill_spi(struct bpf_insn *insn) +{ + return -insn->off / BPF_REG_SIZE - 1; +} + +static u32 off_to_reg(struct bpf_insn *insn) +{ + return spill_spi(insn) + MAX_BPF_REG; +} + +static bool is_spill_off(int off) +{ + return off % BPF_REG_SIZE == 0 && + off <= -BPF_REG_SIZE && + off >= -MAX_BPF_STACK_JIT; +} + +static void mark_opaque(struct env *e, int off, int size) +{ + int b, spi; + + for (b = off; b < off + size; b++) { + if (b >= 0 || b < -MAX_BPF_STACK_JIT) + continue; + spi = (-b - 1) / BPF_REG_SIZE; + e->reg2expr[MAX_BPF_REG + spi] = OPAQUE_EXPR_ID; + } +} + +static int mk_spill(struct scev *scev, u8 size, int id) +{ + switch (size) { + case BPF_B: return expr1(scev, SPILL8, id); + case BPF_H: return expr1(scev, SPILL16, id); + case BPF_W: return expr1(scev, SPILL32, id); + case BPF_DW: return id; + } + return UNKNOWN_EXPR_ID; +} + +static int mk_fill(struct scev *scev, int id, u8 code) +{ + bool sx = BPF_MODE(code) == BPF_MEMSX; + +#ifdef __BIG_ENDIAN + /* A narrow fill from a DW spill reads high bits on big-endian. */ + if (BPF_SIZE(code) != BPF_DW) + return OPAQUE_EXPR_ID; +#endif + + switch (BPF_SIZE(code)) { + case BPF_B: return expr1(scev, sx ? SEXT8 : ZEXT8, id); + case BPF_H: return expr1(scev, sx ? SEXT16 : ZEXT16, id); + case BPF_W: return expr1(scev, sx ? SEXT32 : ZEXT32, id); + case BPF_DW: return sx ? UNKNOWN_EXPR_ID : id; /* DW sign-extended load is invalid */ + } + return UNKNOWN_EXPR_ID; +} + +static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int id) +{ + u8 size = BPF_SIZE(insn->code); + + if (insn->dst_reg != BPF_REG_FP) + return 0; + if (is_spill_off(insn->off)) + return replace_reg(scev, e, off_to_reg(insn), mk_spill(scev, size, id)); + mark_opaque(e, insn->off, bpf_size_to_bytes(size)); + return 0; +} + +static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn) +{ + int id; + + if (insn->src_reg == BPF_REG_FP && is_spill_off(insn->off)) + id = mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code); + else + id = OPAQUE_EXPR_ID; + return replace_reg(scev, e, insn->dst_reg, id); +} + +static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) +{ + const bool little_endian = htons(0x3412) == 0x1234; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u32 *reg2expr = e->reg2expr; + u8 class = BPF_CLASS(insn->code); + u8 x_or_k = BPF_SRC(insn->code); + u8 opcode = BPF_OP(insn->code); + u8 mode = BPF_MODE(insn->code); + u32 dst = insn->dst_reg; + u32 src = insn->src_reg; + u32 op, sext; + bool need_bswap; + int i, id; + s64 imm; + + switch (class) { + case BPF_ALU: + case BPF_ALU64: + switch (opcode) { + case BPF_MOV: + switch (insn->off) { + case 0: sext = 0; break; + case 8: sext = SEXT8; break; + case 16: sext = SEXT16; break; + case 32: sext = SEXT32; break; + default: + goto mark_dst_unknown; + } + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0 && insn->off == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + if (sext) + id = expr1(scev, sext, id); + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + if (opcode == BPF_DIV && insn->off == 1) + op = SDIV; + else if (opcode == BPF_MOD && insn->off == 1) + op = SMOD; + else if (insn->off == 0) + op = opcode; + else + goto mark_dst_unknown; + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + id = expr2(scev, op, reg2expr[dst], id); + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_NEG: + if (src == 0 && insn->off == 0 && insn->imm == 0) + id = expr1(scev, BPF_NEG, reg2expr[dst]); + else + goto mark_dst_unknown; + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_END: + if (src != 0 || insn->off != 0 || (class == BPF_ALU64 && x_or_k != 0)) + goto mark_dst_unknown; + + need_bswap = class == BPF_ALU64 || ((x_or_k == BPF_TO_LE) != little_endian); + switch (insn->imm) { + case 16: op = need_bswap ? BSWAP16 : ZEXT16; break; + case 32: op = need_bswap ? BSWAP32 : ZEXT32; break; + case 64: op = need_bswap ? BSWAP64 : 0; break; + default: + goto mark_dst_unknown; + } + + id = reg2expr[dst]; + if (op) + id = expr1(scev, op, reg2expr[dst]); + + return replace_reg(scev, e, dst, id); + default: + goto mark_dst_unknown; + } + break; + case BPF_LDX: + switch (mode) { + case BPF_MEM: + case BPF_MEMSX: + return maybe_load_fp(scev, e, insn); + default: + goto mark_dst_unknown; + } + case BPF_STX: + switch (mode) { + case BPF_MEM: + return maybe_store_fp(scev, e, insn, reg2expr[src]); + case BPF_ATOMIC: + if (insn->imm == BPF_LOAD_ACQ) + return maybe_load_fp(scev, e, insn); + if (insn->imm == BPF_STORE_REL) { + return maybe_store_fp(scev, e, insn, reg2expr[src]); + } + /* + * verifier does not track other atomic ops precisely, + * hence mark the results as opaque. + */ + if (insn->dst_reg == BPF_REG_FP) + mark_opaque(e, insn->off, bpf_size_to_bytes(BPF_SIZE(insn->code))); + if (insn->imm == BPF_CMPXCHG) + return replace_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID); + if (insn->imm & BPF_FETCH) + return replace_reg(scev, e, src, OPAQUE_EXPR_ID); + break; + } + break; + case BPF_ST: + if (insn->dst_reg == BPF_REG_FP) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + return maybe_store_fp(scev, e, insn, id); + } + break; + case BPF_JMP: + case BPF_JMP32: + if (opcode == BPF_CALL) + forget_call_regs(e); + /* for non-CALL there are no changes in register states */ + break; + case BPF_LD: + switch (mode) { + case BPF_IMM: + /* rX = imm ll */ + if (BPF_SIZE(insn->code) == BPF_DW && insn->src_reg == 0) { + imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; + return replace_reg(scev, e, dst, imm_expr(scev, imm)); + } + /* map, map value, BTF id, function */ + if (BPF_SIZE(insn->code) == BPF_DW) + return replace_reg(scev, e, dst, OPAQUE_EXPR_ID); + goto mark_dst_unknown; + case BPF_ABS: + case BPF_IND: + forget_call_regs(e); + break; + default: + goto mark_dst_unknown; + } + break; + default: + /* unknown instruction, nuke state */ + for (i = 0; i < REGS_NUM; i++) + reg2expr[i] = UNKNOWN_EXPR_ID; + break; + } + return 0; + +mark_dst_unknown: + reg2expr[dst] = UNKNOWN_EXPR_ID; + return 0; +} + +/* + * Construct minimal 'ANY' expression by traversing 'a' and 'b' + * and accumulating non-duplicated non-ANY entries. + */ +static int mk_any(struct scev *scev, int a, int b) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 i, j, ids_buf_sz; + u32 roots[2] = {a, b}; + int id; + + if (a < 0) + return a; + if (b < 0) + return b; + ids_buf_sz = 0; + for (i = 0; i < ARRAY_SIZE(roots); i++) { + scev->stack_sz = 0; + expr_stack_push(scev, roots[i]); + while (scev->stack_sz) { + elt = &scev->expr_stack[--scev->stack_sz]; + id = elt->id; + expr = &scev->exprs[elt->id]; + if (expr->op == ANY) { + if (!expr_stack_push(scev, expr->params[1]) || + !expr_stack_push(scev, expr->params[0])) + return UNKNOWN_EXPR_ID; + } else { + for (j = 0; j < ids_buf_sz; j++) { + if (same_exprs(scev, scev->ids_buf[j], id)) + goto next; + } + if (ids_buf_sz == ARRAY_SIZE(scev->ids_buf)) + return UNKNOWN_EXPR_ID; + scev->ids_buf[ids_buf_sz++] = id; + } +next:; + } + } + if (WARN_ON(ids_buf_sz == 0)) + return -EFAULT; + id = scev->ids_buf[0]; + for (i = 1; i < ids_buf_sz; i++) { + id = expr2(scev, ANY, id, scev->ids_buf[i]); + if (id < 0) + return id; + } + return id; +} + +static int join(struct scev *scev, struct env *acc, struct env *cur) +{ + int i, id; + + if (acc->empty) { + memcpy(acc, cur, sizeof(*acc)); + acc->empty = false; + return 0; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[i])) { + id = mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]); + if (id < 0) + return id; + acc->reg2expr[i] = id; + } + } + return 0; +} + +/* Mark any register modified in 'e' as unknown in 'acc'. */ +static void forget_modified_regs(struct scev *scev, struct env *acc, struct env *e) +{ + u32 i, reg; + + for (i = 0; i < REGS_NUM; i++) { + if (e && is_reg(scev, e->reg2expr[i], ®) && reg == i) + continue; + acc->reg2expr[i] = UNKNOWN_EXPR_ID; + } +} + +/* For a loop H at `header`, mark any register modified as a result of H execution as unknown. */ +static void forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[header].loop; + struct scev *scev = env->scev; + int i, idx, h; + + acc->empty = false; + forget_modified_regs(scev, acc, find_header_env(scev, header)); + for (i = 0; i < loop->exits_cnt; i++) { + idx = loop->exits[i].from; + h = bpf_loop_at_index(env, idx); + /* + * In order to account for situations like this: + * + * header: + * do { + * saved = rX; + * rX = -1000; + * do { // H + * if (random()) + * goto out; // E + * } while (inner_again); + * rX = saved; + * } while (outer_again); + * out: + * + * For exits originating in nested loops check that rX is invariant + * both at the header (H) and at exit (E). + */ + while (h != header) { + forget_modified_regs(scev, acc, find_header_env(scev, h)); + forget_modified_regs(scev, acc, find_loop_env(scev, h, idx)); + idx = h; + h = aux[h].loop_header; + } + /* + * For exits originating at the same loop the header is already accounted for, + * check only the exit environment. + */ + forget_modified_regs(scev, acc, find_loop_env(scev, header, idx)); + } +} + +/* Order worklist in reverse post-order. */ +static bool reverse_postorder_less(const void *a, const void *b, void *arg) +{ + const int *rank = arg; + + return rank[*(const int *)a] > rank[*(const int *)b]; +} + +static const struct min_heap_callbacks worklist_cb = { + .less = reverse_postorder_less, +}; + +static int worklist_push(struct bpf_verifier_env *env, int loop_header, int idx) +{ + struct scev *scev = env->scev; + struct scev_worklist *worklist = &scev->worklist; + u32 tag = (u32)loop_header + 1; + + if (scev->discovered[idx] == tag) + return 0; + + if (min_heap_full_inline(worklist)) { + size_t size = worklist->size ? worklist->size * 2 : 16; + int *data; + + data = kvrealloc(worklist->data, array_size(size, sizeof(*data)), + GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + worklist->data = data; + worklist->size = size; + } + if (!min_heap_push_inline(worklist, &idx, &worklist_cb, env->cfg.postorder_nums)) + return -EFAULT; + + scev->discovered[idx] = tag; + return 0; +} + +static bool reg_alive_at_succ(struct bpf_verifier_env *env, u32 r, u32 insn_idx) +{ + struct bpf_iarray *succ = bpf_insn_successors(env, insn_idx); + u32 i; + + for (i = 0; i < succ->cnt; i++) + if (reg_alive_at(env, r, succ->items[i])) + return true; + return false; +} + +enum changes_at { + LOG_AT_TRANSFER, + LOG_AT_JOIN, +}; + +static void log_env_changes(struct bpf_verifier_env *env, enum changes_at at, + struct env *old, struct env *new, int insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + u64 null_pos = log->end_pos; + u32 old_id, new_id, regno; + bool any_changes = false; + bool is_self_scev; + u64 len; + int r; + + bpf_log(log, "%s %4d: ", at == LOG_AT_TRANSFER ? "t" : "j", insn_idx); + bpf_verbose_insn(env, &env->prog->insnsi[insn_idx]); + len = log->end_pos - null_pos; + bpf_log(log, "%*s", max(37 - (int)len, 1), " "); + bpf_log(log, " ; "); + for (r = 0; r < REGS_NUM; r++) { + old_id = old->reg2expr[r]; + new_id = new->reg2expr[r]; + if (at == LOG_AT_TRANSFER && !reg_alive_at_succ(env, r, insn_idx)) + continue; + if (at == LOG_AT_JOIN && !reg_alive_at(env, r, insn_idx)) + continue; + if (same_exprs(scev, old_id, new_id)) + continue; + is_self_scev = is_reg(scev, new_id, ®no) && r == regno; + if (old->empty && is_self_scev) + continue; + if (any_changes) + bpf_log(log, ", "); + log_reg(env, r); + bpf_log(log, " "); + log_expr(env, old_id); + bpf_log(log, " -> "); + log_expr(env, new_id); + any_changes = true; + } + if (old->empty && !any_changes) + bpf_log(log, "<empty join>"); + bpf_log(log, "\n"); + if (!old->empty && !any_changes) + bpf_vlog_reset(log, null_pos); +} + +static bool is_probe_read_helper(u32 func_id) +{ + return func_id == BPF_FUNC_probe_read || + func_id == BPF_FUNC_probe_read_kernel || + func_id == BPF_FUNC_probe_read_user || + func_id == BPF_FUNC_probe_read_str || + func_id == BPF_FUNC_probe_read_kernel_str || + func_id == BPF_FUNC_probe_read_user_str; +} + +/* + * If instruction is an indirect write to stack, invalidate SCEVs for spi's + * that this instruction can touch. + */ +static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u8 class = BPF_CLASS(insn->code); + const unsigned long *mask; + u32 spi, reg; + bool opaque; + + /* Direct fp stores are fine. */ + if ((class == BPF_ST || class == BPF_STX) && insn->dst_reg == BPF_REG_FP) + return; + + mask = bpf_may_write_mask(env, idx); + opaque = bpf_helper_call(insn) && is_probe_read_helper(insn->imm); + for_each_set_bit(spi, mask, MAX_BPF_STACK_SLOTS) { + reg = MAX_BPF_REG + spi; + if (cur_env->reg2expr[reg] == UNKNOWN_EXPR_ID) + continue; + replace_reg(scev, cur_env, reg, opaque ? OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID); + } +} + +/* + * Collect loop-entry registers referenced by expr 'id', best effort. + * DEPTH_LIMIT may leave dependencies unrecorded. This mask only avoids + * widening that would lose stack-access precision; missing dependencies + * may cause false rejections, but this is not a soundness issue. + */ +static void or_expr_regs(struct bpf_verifier_env *env, u32 id, unsigned long *mask) +{ + struct scev *scev = env->scev; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->exprs[id].op == REG) + __set_bit(scev->exprs[id].params[0], mask); + } +} + +/* Mask of argument registers (R1..R5) a call at 'idx' passes by register. */ +static u16 call_params_mask(struct bpf_verifier_env *env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct bpf_call_summary cs; + int n = bpf_get_call_summary(env, insn, &cs) ? cs.arg_slot_cnt : MAX_BPF_FUNC_REG_ARGS; + + return n ? GENMASK(BPF_REG_1 + n - 1, BPF_REG_1) : 0; +} + +/* + * For instructions like: + * - *(u64 *)(rBase + off) = rX + * - rX = *(u64 *)(rBase + off) + * - calls that construct objects on stack (e.g. dynptr_from_mem(rBase, ...)) + * When 'rBase' can be a stack pointer and is derived from some registers Rs + * defined at loop entry, record Rs into 'mask'. + */ +static void collect_store_base_regs(struct bpf_verifier_env *env, + struct env *cur_env, int idx, unsigned long *mask) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + u8 class = BPF_CLASS(insn->code); + u8 size = BPF_SIZE(insn->code); + u16 base_regs = 0; + u32 r; + + if (size == BPF_W || size == BPF_DW) { + if ((class == BPF_STX || class == BPF_ST) && insn->dst_reg != BPF_REG_FP) + base_regs |= BIT(insn->dst_reg); + else if (class == BPF_LDX && insn->src_reg != BPF_REG_FP) + base_regs |= BIT(insn->src_reg); + } + + if (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL && + bpf_needs_fixed_stack_off(env, idx)) + base_regs |= call_params_mask(env, idx); + + base_regs &= aux[idx].stack_ptrs; + for (r = 0; r < MAX_BPF_REG; r++) + if (base_regs & BIT(r)) + or_expr_regs(env, cur_env->reg2expr[r], mask); +} + +/* Find the topmost loop header containing idx inside cur_header, or -1 if none. */ +static int topmost_nested_loop(struct bpf_verifier_env *env, int idx, int cur_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int header; + + for (header = bpf_loop_at_index(env, idx); header >= 0; + header = aux[header].loop_header) + if (aux[header].loop_header == cur_header) + return header; + + return -1; +} + +/* + * Join cur_env into the successor's environment and schedule their traversal. + * Map successors in nested loops to their topmost nested header. + * Ignore successors outside the current loop and its nested loops. + */ +static int join_successor(struct bpf_verifier_env *env, int cur_header, int succ_idx, + struct env *old_env, struct env *cur_env) +{ + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + struct scev *scev = env->scev; + int err, nested_header; + struct env *succ_env; + + /* + * There are several possibilities for a successor: + * - succ_idx can be a part of a loop outside of the cur_header's loop, + * such edges are ignored. + * - succ_idx can be a part of the same loop as cur_header, + * for such edges succ_idx environment is updated: + * e[succ_idx] = join(e[succ_idx], cur_env) + * - succ_idx can be a part of some loop inner to cur_header, + * in such a case there exists some loop header H, + * such that H.loop_header == cur_header + * and H is the same as succ_idx's loop or contains it. + */ + if (bpf_loop_at_index(env, succ_idx) != cur_header) { + /* + * topmost_nested_loop() either finds H or returns -1, + * in case if succ_idx is a part of a loop outer to cur_header. + */ + nested_header = topmost_nested_loop(env, succ_idx, cur_header); + if (nested_header < 0) + return 0; + succ_idx = nested_header; + } + succ_env = get_loop_env(scev, cur_header, succ_idx); + if (!succ_env) + return -ENOMEM; + if (log_level2) + memcpy(old_env, succ_env, sizeof(*old_env)); + err = join(scev, succ_env, cur_env); + if (err) + return err; + err = worklist_push(env, cur_header, succ_idx); + if (err) + return err; + if (log_level2) + log_env_changes(env, LOG_AT_JOIN, old_env, succ_env, succ_idx); + return 0; +} + +/* Unsupported loops get only an unknown header environment. */ +static bool can_compute_loop_scev(const struct bpf_loop *loop) +{ + return !loop->irreducible && !loop->exits_overflow; +} + +static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) +{ + struct scev_worklist *worklist = &env->scev->worklist; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *cur_loop = aux[cur_header].loop; + struct env *header_env; + struct scev *scev = env->scev; + struct bpf_loop *nested_loop; + struct env *cur_env = NULL; + struct env *old_env = NULL; + struct bpf_iarray *succ; + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + int s, i, err, idx, succ_idx; + u32 r; + + if (log_level2) + bpf_log(&env->log, "Computing SCEV for loop at %d:\n", cur_header); + + /* + * For irreducible loops, and for loops nesting a loop with a truncated + * exit list, just assume that everything is clobbered for now. + */ + if (!can_compute_loop_scev(cur_loop)) { + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + return -ENOMEM; + /* The freshly allocated environment has all expressions unknown. */ + bitmap_fill(cur_loop->store_base_regs, REGS_NUM); + header_env->empty = false; + return 0; + } + + cur_env = kzalloc(sizeof(*cur_env), GFP_KERNEL_ACCOUNT); + if (!cur_env) + goto nomem; + if (log_level2) { + old_env = kzalloc(sizeof(*old_env), GFP_KERNEL_ACCOUNT); + if (!old_env) + goto nomem; + } + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + goto nomem; + err = setup_initial_loop_env(env, header_env, cur_header); + if (err) + goto out; + err = worklist_push(env, cur_header, cur_header); + if (err) + goto out; + + while (worklist->nr) { + idx = *min_heap_peek_inline(worklist); + min_heap_pop_inline(worklist, &worklist_cb, env->cfg.postorder_nums); + + /* join_successor() maps nested-loop entries to their representative header. */ + nested_loop = idx != cur_header ? aux[idx].loop : NULL; + memcpy(cur_env, find_loop_env(scev, cur_header, idx), sizeof(*cur_env)); + if (nested_loop) { + /* + * Process nested loop as a single instruction by + * forgetting anything non-invariant in the nested loop + */ + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + /* Pull the nested loop's stack-store base dependencies up. */ + for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM) + or_expr_regs(env, cur_env->reg2expr[r], cur_loop->store_base_regs); + forget_non_invariants(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + /* + * Treat nested loop exits as successors, + * join cur_env into successor's envs. + */ + for (i = 0; i < nested_loop->exits_cnt; i++) { + s = nested_loop->exits[i].to; + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } else { + /* + * Iterate instructions within a single basic block + * starting at 'idx' mutating 'cur_env'. + */ + for (;;) { + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + collect_store_base_regs(env, cur_env, idx, cur_loop->store_base_regs); + err = transfer(env, cur_env, idx); + if (err) + goto out; + reset_scevs_at_indirect_writes(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + succ = bpf_insn_successors(env, idx); + if (succ->cnt != 1) + break; + succ_idx = succ->items[0]; + if (aux[idx].bb_end || aux[succ_idx].need_scev || + bpf_loop_at_index(env, succ_idx) != cur_header) + break; + idx = succ_idx; + } + /* Join cur_env into basic block successor's envs. */ + iarray_for_each(s, succ) { + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } + } + + err = 0; +out: + kfree(cur_env); + kfree(old_env); + return err; +nomem: + err = -ENOMEM; + goto out; +} + +static void mark_latches(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop; + int len = env->prog->len; + int i, j, k, from; + + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + aux[i].need_scev = true; + if (!can_compute_loop_scev(loop)) + continue; + /* + * Make sure to only set .need_scev for exits and latches + * belonging to the loop at `i`. If can_compute_loop_scev() + * fails for a loop, it should hide it's instructions from + * compute_insn_scevs(). + */ + for (j = 0; j < loop->backedges_cnt; j++) + for (k = 0; k < loop->backedges[j].latches_cnt; k++) + aux[loop->backedges[j].latches[k]].need_scev = true; + for (j = 0; j < loop->exits_cnt; j++) { + from = loop->exits[j].from; + if (bpf_loop_at_index(env, from) == i) + aux[from].need_scev = true; + } + } +} + +static bool is_any_imm_reg_opaque(struct scev *scev, u32 id) +{ + u32 l, r, reg, order; + s64 imm; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || + is_imm(scev, id, &imm) || + is_reg(scev, id, ®) || + is_opaque(scev, id) || + is_any(scev, id, &l, &r)) + continue; + return false; + } + return true; +} + +/* + * Can implement explicit stack version, but it is harder to read. + * Stick with recursive version for now. + */ +static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + struct expr expr; + int p0, p1, id; + + if (lvl >= EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + + expr = scev->exprs[root]; /* snapshot the expr before potential realloc */ + switch (op_params_num(expr.op)) { + case 0: + id = root; + break; + case 1: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + id = expr1(scev, expr.op, p0); + break; + case 2: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + p1 = transform_expr_once(scev, lvl + 1, expr.params[1], priv, fn); + id = expr2(scev, expr.op, p0, p1); + break; + } + return id < 0 ? id : fn(scev, id, priv); +} + +static int transform_expr(struct scev *scev, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + int i = 0, id_old, id_new = root; + + do { + /* + * The depth limit is somewhat arbitrary and is essentially a failsafe. + * oscillation should not really happen. Still, it seems harsh to error out. + */ + if (i++ == 4 * EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + id_old = id_new; + id_new = transform_expr_once(scev, 0, id_old, priv, fn); + if (id_new < 0) + return id_new; + } while (id_old != id_new); + return id_new; +} + +static int simplify(struct scev *scev, u32 id, void *priv) +{ + int a, b, l, r, t, base, slope; + s64 imm1, imm2; + + /* (+ (linear base slope) imm) -> (linear (+ base imm) slope) */ + if (is_add(scev, id, &l, &r) && + is_linear(scev, l, &base, &slope) && + is_imm(scev, r, &imm1)) + return expr2(scev, LINEAR_SCEV, expr2(scev, BPF_ADD, base, r), slope); + + /* (+ imm imm) -> imm */ + if (is_add(scev, id, &l, &r) && + is_imm(scev, l, &imm1) && + is_imm(scev, r, &imm2)) + return imm_expr(scev, imm1 + imm2); + + /* (zext32 (imm X)) -> (imm (u32)X) */ + if (is_zext32(scev, id, &l) && is_imm(scev, l, &imm1)) + return imm_expr(scev, (u64)(u32)imm1); + + /* + * (zext32 (+ (zext32 $a) $b)) -> (zext32 (+ $a $b)) + * (zext32 (+ $a (zext32 $b))) -> (zext32 (+ $a $b)) + */ + if ((is_zext32(scev, id, &t) && is_add(scev, t, &l, &b) && is_zext32(scev, l, &a)) || + (is_zext32(scev, id, &t) && is_add(scev, t, &a, &r) && is_zext32(scev, r, &b))) { + t = expr2(scev, BPF_ADD, a, b); + t = expr1(scev, ZEXT32, t); + return t; + } + + return id; +} + +static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header_env) +{ + struct scev *scev = env->scev; + int l, r, t, id, ra, rb, ra_expr; + s64 imm; + + /* + * Note: + * reg2expr[ra] describes a change in rA's value over a single loop iteration. + * It does not account for rA's entry value. However, the SCEV constructed for rA + * must include an additional rA entry edge. For example: + * + * r0 = ..., r1 = 7 + * l: if r0 == 10 goto e // reg2expr[r1] == (imm 42) => reg2scev[r1] == (any r1 42) + * r0 += 1 + * r1 = 42 + * goto l + * e: + * ... at exit r1 is in {7,42} ... + */ + for (ra = 0; ra < REGS_NUM; ra++) { + ra_expr = transform_expr(scev, header_env->reg2expr[ra], NULL, simplify); + if (ra_expr < 0) + return ra_expr; + header_env->reg2expr[ra] = ra_expr; + /* rA = (+ rA IMM) -> (linear rA IMM) */ + if (is_add(scev, ra_expr, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = (is_zext32 (+ rA IMM)) -> (is_zext32 (linear rA IMM)) */ + if (is_zext32(scev, ra_expr, &t) && + is_add(scev, t, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + id = expr1(scev, ZEXT32, id); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = rA -> rA*/ + if (is_reg(scev, ra_expr, &rb) && ra == rb) { + header_env->reg2scev[ra] = ra_expr; + continue; + } + /* rA = (any 1 2 3 4 ...) -> (any rA 1 2 3 4 ...) */ + if (is_any_imm_reg_opaque(scev, ra_expr)) { + id = mk_any(scev, expr1(scev, REG, ra), ra_expr); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + + } + return 0; +} +static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) +{ + u32 reg, *reg2scev = priv; + + /* (reg r) -> ... equation at header ... */ + if (is_reg(scev, id, ®)) + return reg2scev[reg]; + /* keep expression shapes */ + return id; +} + +static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, struct env *einsn) +{ + struct scev *scev = env->scev; + int id, reg; + + for (reg = 0; reg < REGS_NUM; reg++) { + id = einsn->reg2expr[reg]; + id = transform_expr_once(scev, 0, id, eheader->reg2scev, instantiate_header_scevs); + if (id < 0) + return id; + id = transform_expr(scev, id, NULL, simplify); + if (id < 0) + return id; + einsn->reg2scev[reg] = id; + } + + return 0; +} + +static int log_scevs(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct env *header_env, *latch_env; + struct scev *scev = env->scev; + struct bpf_loop *loop; + int i, j, k, len, latch; + + len = env->prog->len; + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + header_env = find_header_env(scev, i); + bpf_log(log, "scev at header %d:\n", i); + print_env(env, header_env, i); + if (!can_compute_loop_scev(loop)) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + for (k = 0; k < loop->backedges[j].latches_cnt; k++) { + latch = loop->backedges[j].latches[k]; + bpf_log(log, " scev at %s %d:\n", k ? "exit" : "latch", latch); + latch_env = find_loop_env(scev, i, latch); + if (verifier_bug_if(!latch_env, env, "i=%d, latch=%d", i, latch)) + return -EFAULT; + print_env(env, latch_env, latch); + } + } + } + return 0; +} + +int bpf_compute_scev(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct scev *scev = env->scev; + int *postorder = env->cfg.insn_postorder; + int cnt = env->cfg.cur_postorder; + int i, idx, err, header; + + mark_latches(env); + /* + * Visit loop headers in postorder, to guarantee that scevs + * for innermost loops are computed first. + */ + for (i = 0; i < cnt; i++) { + idx = postorder[i]; + if (!aux[idx].loop) + continue; + err = compute_scev_for_loop(env, idx); + if (err) + return err; + } + + /* + * Compute scevs from exprs collected on a previous step. Iterate instructions in + * reverse post-order so that each loop header is processed before instructions + * reachable from it. + */ + for (i = cnt - 1; i >= 0; i--) { + idx = postorder[i]; + if (!aux[idx].need_scev) + continue; + header = bpf_loop_at_index(env, idx); + err = aux[idx].loop + ? compute_header_scevs(env, find_header_env(scev, header)) + : compute_insn_scevs(env, + find_header_env(scev, header), + find_loop_env(scev, header, idx)); + if (err) + return err; + } + + for (i = 0; i < env->prog->len; i++) + if (aux[i].loop) + aux[i].prune_point = true; + + if (env->log.level & BPF_LOG_LEVEL2) { + err = log_scevs(env); + if (err) + return err; + } + + return 0; +} + +void bpf_free_scev(struct bpf_verifier_env *env) +{ + struct scev *scev = env->scev; + struct insn_envs *envs; + int i; + u32 j; + + if (!scev) + return; + for (i = 0; i < scev->envs_cnt; i++) { + envs = scev->envs[i]; + if (!envs) + continue; + for (j = 0; j < envs->cnt; j++) + kfree(envs->entries[j].env); + kfree(envs); + } + if (scev->exprs_ht) + for (i = 0; i < scev->exprs_ht_cnt; i++) + kvfree(scev->exprs_ht[i]); + kvfree(scev->exprs_ht); + kvfree(scev->worklist.data); + kvfree(scev->envs); + kvfree(scev->exprs); + kvfree(scev->discovered); + kfree(scev); + env->scev = NULL; +} + +int bpf_init_scev(struct bpf_verifier_env *env) +{ + struct scev *scev; + + scev = kzalloc(sizeof(struct scev), GFP_KERNEL_ACCOUNT); + if (!scev) + return -ENOMEM; + env->scev = scev; + bpf_mark_reg_known_scalar(&scev->zero_reg, 0); + bpf_mark_reg_unknown_imprecise(&scev->unknown_reg); + /* A power of two for hash & (exprs_ht_cnt - 1)` indexing. */ + scev->exprs_ht_cnt = roundup_pow_of_two(max(256U, DIV_ROUND_UP(env->prog->len, 4))); + scev->exprs_ht = kvcalloc(scev->exprs_ht_cnt, sizeof(*scev->exprs_ht), GFP_KERNEL_ACCOUNT); + if (!scev->exprs_ht) + goto nomem; + if (expr0(scev, UNKNOWN) < 0 || expr0(scev, OPAQUE) < 0) + goto nomem; + scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); + scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); + if (!scev->envs || !scev->discovered) + goto nomem; + /* + * Remember original program length, in case bpf_free_scev() + * is called after bpf program rewrites that increase program + * length. + */ + scev->envs_cnt = env->prog->len; + return 0; +nomem: + bpf_free_scev(env); + return -ENOMEM; +} + +static struct bpf_stack_state *scev_regno_to_stack(const struct bpf_func_state *st, u32 r) +{ + int spi, slots_available; + + if (r < MAX_BPF_REG) + return NULL; + + slots_available = st->allocated_stack / BPF_REG_SIZE; + spi = r - MAX_BPF_REG; + if (spi < slots_available) + return &st->stack[spi]; + + return NULL; +} + +static struct bpf_reg_state *scev_regno_to_reg(struct bpf_func_state *st, u32 r) +{ + return r < MAX_BPF_REG ? &st->regs[r] : NULL; +} + +/* Linear widening requires a physical register or a full spill. */ +static struct bpf_reg_state *scev_reg_for_linear(struct bpf_func_state *st, u32 r) +{ + struct bpf_stack_state *stack; + + if (r < MAX_BPF_REG) + return scev_regno_to_reg(st, r); + stack = scev_regno_to_stack(st, r); + if (!stack || bpf_spill_size(stack) != BPF_REG_SIZE) + return NULL; + return &stack->spilled_ptr; +} + +/* + * Return a borrowed value; never use it as a widening destination. + * NULL denotes no readable value. + */ +static const struct bpf_reg_state * +scev_reg_for_read(struct bpf_verifier_env *env, const struct bpf_func_state *st, u32 r) +{ + const struct bpf_stack_state *stack; + bool zero = true; + int i; + + if (r < MAX_BPF_REG) + return st->regs[r].type == NOT_INIT ? NULL : &st->regs[r]; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return NULL; + + stack = scev_regno_to_stack(st, r); + if (!stack) + return &env->scev->unknown_reg; + if (bpf_spill_size(stack) == BPF_REG_SIZE) + return &stack->spilled_ptr; + + /* Partial spills do not describe the value of the whole slot. */ + for (i = 0; i < BPF_REG_SIZE; i++) { + switch (stack->slot_type[i]) { + case STACK_ZERO: + break; + case STACK_INVALID: + case STACK_POISON: + case STACK_MISC: + zero = false; + break; + case STACK_SPILL: + if (!bpf_is_spilled_scalar_reg(stack)) + return NULL; + zero = false; + break; + default: + return NULL; + } + } + return zero ? &env->scev->zero_reg : &env->scev->unknown_reg; +} + +/* Materialize a writable value in the current state, growing its stack if needed. */ +static int scev_reg_for_write(struct bpf_verifier_env *env, struct bpf_func_state *st, + u32 r, struct bpf_reg_state **out) +{ + const struct bpf_reg_state *value; + struct bpf_stack_state *stack; + u32 spi; + int err; + + *out = scev_regno_to_reg(st, r); + if (*out) + return 0; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return -EINVAL; + + spi = r - MAX_BPF_REG; + err = bpf_grow_stack_state(env, st, (spi + 1) * BPF_REG_SIZE); + if (err) + return err; + + /* Resolve pointers after growth, which may relocate the stack. */ + value = scev_reg_for_read(env, st, r); + if (!value) + return -EINVAL; + stack = scev_regno_to_stack(st, r); + stack->spilled_ptr = *value; + memset(stack->slot_type, STACK_SPILL, sizeof(stack->slot_type)); + *out = &stack->spilled_ptr; + return 0; +} + +static bool scev_reg_alive(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 r) +{ + int insn_idx = bpf_frame_insn_idx(st, st->curframe); + u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before; + int spi; + + if (r < MAX_BPF_REG) { + return BIT(r) & live_regs; + } else { + spi = r - MAX_BPF_REG; + return bpf_stack_slot_alive(env, st->curframe, spi * 2) || + bpf_stack_slot_alive(env, st->curframe, spi * 2 + 1); + } +} + +/* + * Latch is a condition deciding if execution remains inside a loop. + * Linear latch represents a condition 'if <reg> <op> <loop invariant> goto <loop-header>', + * where equation '<base> + i * <step> <op> <bound>' describes values taken by register <reg>, + * 'i' is the loop iteration number, starting from 0. + * 'zext32' indicates the latch is for (zext32 (linear ...)) SCEV. + * 'jmp32' indicates that the latch is a JMP32 instruction. + */ +struct linear_latch { + u32 insn_idx; + u32 base_expr; + u32 step_expr; + u32 bound_expr; + u8 op; + bool zext32; + bool jmp32; +}; + +static bool loop_invariant(struct scev *scev, u32 id) +{ + s64 imm; + u32 reg; + + return is_reg(scev, id, ®) || is_imm(scev, id, &imm); +} + +static int match_linear_latch(struct bpf_verifier_env *env, + u32 header, + u32 latch_idx, + struct linear_latch *latch) +{ + struct bpf_insn *insn = &env->prog->insnsi[latch_idx]; + struct scev *scev = env->scev; + struct env *latch_env; + u32 true_branch_loop; + u32 true_branch_tgt; + u32 src_reg_scev; + u32 dst_reg_scev; + u32 t, l, r, op; + int id; + + op = BPF_OP(insn->code); + latch->jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; + /* Flip the condition if true branch jumps out of the loop. */ + true_branch_tgt = latch_idx + bpf_jmp_offset(insn) + 1; + true_branch_loop = bpf_loop_at_index(env, true_branch_tgt); + if (true_branch_loop != header && + !bpf_is_nested_loop(env, true_branch_loop, header)) + op = bpf_rev_opcode(op); + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + case BPF_JLT: + case BPF_JLE: + case BPF_JGT: + case BPF_JGE: + case BPF_JNE: + break; + default: + return false; + } + + latch->insn_idx = latch_idx; + + latch_env = find_loop_env(scev, header, latch_idx); + if (verifier_bug_if(!latch_env, env, "header_idx=%d, latch_idx=%d\n", header, latch_idx)) + return -EFAULT; + dst_reg_scev = latch_env->reg2scev[insn->dst_reg]; + src_reg_scev = latch_env->reg2scev[insn->src_reg]; + /* + * Make sure (linear ...) is in dst_reg_scev: + * 'if <invariant> <op> <linear>' ⇔ + * 'if <linear> <flip(op)> <invariant>'. + */ + if (BPF_SRC(insn->code) == BPF_X && + is_linear_or_linear32(scev, src_reg_scev, &l, &r, &latch->zext32)) { + swap(dst_reg_scev, src_reg_scev); + op = bpf_flip_opcode(op); + } + latch->op = op; + /* + * simplify() produces two shapes for linear latches: + * - (linear register step) + * ^----------------------------+-- base_expr + * base_expr | + * - (linear (+ register imm) step) | + * ^----------------------------' + */ + if (!is_linear_or_linear32(scev, dst_reg_scev, + &latch->base_expr, &latch->step_expr, &latch->zext32)) + return false; + + if (BPF_SRC(insn->code) == BPF_K) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + latch->bound_expr = id; + } else { + latch->bound_expr = src_reg_scev; + } + + if (!loop_invariant(scev, latch->step_expr) || + !loop_invariant(scev, latch->bound_expr)) + return false; + + /* (linear register step) */ + if (is_reg(scev, latch->base_expr, &t)) + return true; + + /* (linear (+ register imm) step) */ + if (is_add(scev, latch->base_expr, &l, &r) && + is_reg(scev, l, &t) && + loop_invariant(scev, r)) + return true; + + return false; +} + +struct scev_value { + u64 value; /* scalar value or a pointer offset */ + int ptr_reg; /* SCEV register identifying the pointer origin, or -1. */ +}; + +static void log_scev_value(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *v) +{ + const struct bpf_reg_state *reg; + + if (v->ptr_reg >= 0) { + reg = scev_reg_for_read(env, st, v->ptr_reg); + log_reg(env, v->ptr_reg); + bpf_log(&env->log, " (%s)", reg ? reg_type_str(env, reg->type) : "unavailable"); + } else { + bpf_log(&env->log, "scalar %llu", v->value); + } +} + +static void log_incompatible_scev_values(struct bpf_verifier_env *env, const char *reason, + struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + bpf_log(&env->log, "scev: %s ", reason); + log_scev_value(env, st, a); + bpf_log(&env->log, ", "); + log_scev_value(env, st, b); + bpf_log(&env->log, "\n"); +} + +static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + const struct bpf_reg_state *ra, *rb; + + if (a->ptr_reg < 0 || b->ptr_reg < 0) { + if (a->ptr_reg < 0 && b->ptr_reg < 0) + return true; + goto incompatible; + } + ra = scev_reg_for_read(env, st, a->ptr_reg); + rb = scev_reg_for_read(env, st, b->ptr_reg); + if (ra && rb && bpf_same_memory_origin(ra, rb)) + return true; + +incompatible: + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "incompatible latch operands", st, a, b); + return false; +} + +static bool scev_value_add(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b, + struct scev_value *result) +{ + if (a->ptr_reg >= 0 && b->ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "can't add two pointers", st, a, b); + return false; + } + result->value = a->value + b->value; + result->ptr_reg = a->ptr_reg >= 0 ? a->ptr_reg : b->ptr_reg; + return true; +} + +static bool eval_expr(struct bpf_verifier_env *env, struct scev *scev, + struct bpf_func_state *st, u32 id, struct scev_value *result) +{ + struct scev_value lval, rval; + const struct bpf_reg_state *reg; + u32 l, r, regno; + s64 imm; + + if (is_reg(scev, id, ®no)) { + reg = scev_reg_for_read(env, st, regno); + if (!reg || reg->type == NOT_INIT || !tnum_is_const(reg->var_off)) + return false; + result->value = reg->var_off.value; + result->ptr_reg = reg->type == SCALAR_VALUE ? -1 : (int)regno; + return true; + } else if (is_imm(scev, id, &imm)) { + result->value = imm; + result->ptr_reg = -1; + return true; + } else if (is_add(scev, id, &l, &r) && + eval_expr(env, scev, st, l, &lval) && + eval_expr(env, scev, st, r, &rval)) { + return scev_value_add(env, st, &lval, &rval, result); + } + return false; +} + +static bool is_signed_op(u8 op) +{ + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + return true; + default: + return false; + } +} + +static bool compute_backedge_taken_count(u8 op, u64 initial, u64 step, u64 bound, u64 *result) +{ + u64 rem, diff, backedge_taken_count; + bool up, down; + + if (step == 0) + return false; + down = (s64)step < 0; + up = (s64)step > 0; + diff = bound - initial; + switch (op) { + case BPF_JLT: + /* + * E.g.: 3 + 2*i < 7 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. + * Here the inequality describes a set of values `i` might take + * while the latch condition remains true (the backedge is taken). + * The backedge_taken_count is the size of this set. + */ + if (/* + * If the condition is initially true, decreasing values won't + * change that without underflow. If the condition is initially false, + * the main pass would do a single iteration anyway. + */ + down || + /* Is the latch condition false on the first iteration? */ + initial >= bound || + /* Count iterations satisfying the strict inequality, including i=0. */ + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the counter: + * initial + backedge_taken_count * step > U64_MAX + */ + backedge_taken_count > div64_u64(U64_MAX - initial, step)) + return false; + break; + case BPF_JLE: + /* E.g.: 3 + 2*i ≤ 7 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (down || initial > bound || + /* Count iterations satisfying the non-strict inequality. */ + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(U64_MAX - initial, step)) + return false; + break; + case BPF_JGT: + /* E.g.: 7 - 2*i > 3 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. */ + if (up || initial <= bound || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || + /* + * The last iteration underflows the counter if: + * initial + backedge_taken_count * step < 0 ⇔ + * backedge_taken_count * step < -initial ⇔ + * backedge_taken_count > initial / -step + * (direction flips because step < 0) + */ + backedge_taken_count > div64_u64(initial, -step)) + return false; + break; + case BPF_JGE: + /* E.g.: 7 - 2*i ≥ 3 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (up || initial < bound || + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial, -step)) + return false; + break; + case BPF_JSLT: + if (down || (s64)initial >= (s64)bound || + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the signed counter: + * (s64)initial + backedge_taken_count * step > S64_MAX + */ + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) + return false; + break; + case BPF_JSLE: + if (down || (s64)initial > (s64)bound || + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) + return false; + break; + case BPF_JSGT: + if (up || (s64)initial <= (s64)bound || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || + /* + * The last iteration underflows the signed counter if: + * (s64)initial + backedge_taken_count * step < S64_MIN ⇔ + * backedge_taken_count * step < S64_MIN - (s64)initial ⇔ + * backedge_taken_count > initial - (u64)S64_MIN / -step + * (direction flips because step < 0) + */ + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) + return false; + break; + case BPF_JSGE: + if (up || (s64)initial < (s64)bound || + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) + return false; + break; + case BPF_JNE: + /* + * For a decreasing counter flip the diff and step: + * 7 - 2*i ≠ 3 ⇔ 2*i ≠ 7 - 3. + */ + diff = down ? -diff : diff; + step = down ? -step : step; + backedge_taken_count = div64_u64_rem(diff, step, &rem); + if (diff == 0 || rem != 0) + return false; + break; + default: + return false; + } + + *result = backedge_taken_count; + return true; +} + +/* + * Loop with post-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2] header executed 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * if r0 != 3 goto l r0 ∈ [1,2,3] backedge taken 2 times + * ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = 1 + k + * + * Loop with pre-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2,3] header executed 4 times + * if r0 == 3 goto e r0 ∈ [0,1,2,3] backedge taken 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * goto l r0 ∈ [1,2,3] + * e: ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = k + */ +static bool compute_max_iters(struct bpf_verifier_env *env, + struct bpf_func_state *st, + struct linear_latch *latch, + struct bpf_loop_iters *iters) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[bpf_loop_at_index(env, latch->insn_idx)].loop; + struct scev *scev = env->scev; + struct scev_value initial_val, step_val, bound_val; + u64 step, bound, initial, total_change, backedge_taken_count; + u8 op = latch->op; + + if (!eval_expr(env, scev, st, latch->base_expr, &initial_val) || + !eval_expr(env, scev, st, latch->step_expr, &step_val) || + !eval_expr(env, scev, st, latch->bound_expr, &bound_val)) + return false; + + if (step_val.ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) { + bpf_log(&env->log, "scev: latch step is a pointer "); + log_scev_value(env, st, &step_val); + bpf_log(&env->log, "\n"); + } + return false; + } + if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + return false; + + initial = initial_val.value; + step = step_val.value; + bound = bound_val.value; + + /* + * The code below only handles mixed 64-bit/32-bit comparisons/recurrences + * when initial bounds and step fit in 32-bit domain. + */ + if (latch->zext32 || latch->jmp32) { + /* + * With the above constraint doing `l: w0 += 1; if r0 s< 256 goto l;` + * is equivalent to `l: w0 += 1; if r0 < 256 goto l;` + */ + if (latch->jmp32 && is_signed_op(op)) { + initial = (s64)(s32)(u32)initial; + bound = (s64)(s32)(u32)bound; + step = (s64)(s32)(u32)step; + } else { + initial = (u32)initial; + bound = (u32)bound; + step = (s64)(s32)(u32)step; + } + } + if (latch->zext32 ^ latch->jmp32 && (initial != initial_val.value || + bound != bound_val.value || + step != step_val.value)) { + if (env->log.level & BPF_LOG_LEVEL2) + bpf_log(&env->log, + "scev: mixed-width latch has operands outside 32-bit ranges\n"); + return false; + } + + + if (!compute_backedge_taken_count(op, initial, step, bound, &backedge_taken_count)) + return false; + + if (latch->zext32 || latch->jmp32) { + /* + * compute_backedge_taken_count() guarantees that 'total_change' + * does not overflow in 64-bit domain. + */ + total_change = initial + step * backedge_taken_count; + if (latch->jmp32 && is_signed_op(op)) { + if ((s64)total_change < S32_MIN || (s64)total_change > S32_MAX) + return false; + } else { + if (total_change > U32_MAX) + return false; + } + } + + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + + /* The header executes once on entry and once for each taken backedge, hence +1. */ + iters->max_header_count = backedge_taken_count + 1; + /* + * The latch is a conditional jump with one jump target exiting the loop. + * Linear latch is matched only if the loop has a single backedge. + * The loop still, however can have multiple exits. + * In such case, conservatively assume that non-latch exit can happen + * at any iteration, thus setting minimal number of iterations as 0. + */ + iters->min_header_count = loop->exits_cnt == 1 ? iters->max_header_count : 0; + return true; +} + +static void mark_scev_reg_scratched(struct bpf_verifier_env *env, u32 r) +{ + if (r < MAX_BPF_REG) + mark_reg_scratched(env, r); + else + mark_stack_slot_scratched(env, r - __MAX_BPF_REG); +} + +/* + * An unknown scalar is its own widening. + * Stack bytes without a value are represented by scev->unknown_reg. + */ +static bool is_unbound_scev_reg(struct bpf_verifier_env *env, const struct bpf_func_state *st, + u32 r) +{ + const struct bpf_reg_state *reg = scev_reg_for_read(env, st, r); + + return reg && reg->type == SCALAR_VALUE && !reg->id && reg->step <= 1 && + tnum_is_unknown(reg->var_off) && + reg->r64.size == U64_MAX && reg->r32.size == U32_MAX; +} + +/* Main logic in verifier.c forbids varying offsets for certain register types. */ +static bool is_widenable_reg_type(const struct bpf_reg_state *reg) +{ + if (type_may_be_null(reg->type)) + return false; + + switch (base_type(reg->type)) { + case SCALAR_VALUE: + case PTR_TO_MAP_VALUE: + case PTR_TO_MAP_KEY: + case PTR_TO_STACK: + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + case PTR_TO_MEM: + case PTR_TO_BUF: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + +static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) +{ + switch (base_type(reg->type)) { + case SCALAR_VALUE: + reg->id = 0; + break; + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + reg->id = ++env->id_gen; + break; + default: + break; + } +} + +/* Check that ANY leaves can be unioned with r's loop-entry value. */ +static bool is_widenable_any(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct env *header_env, u32 r, u32 id) +{ + const struct bpf_reg_state *reg, *leaf_reg; + struct scev *scev = env->scev; + u32 l, ra, rr, leaf, order; + s64 imm; + + if (!is_any(scev, id, &l, &rr)) + return false; + reg = scev_reg_for_read(env, loop_entry, r); + if (!reg || !is_widenable_reg_type(reg)) + return false; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || is_any(scev, id, &l, &rr)) + continue; + if (is_imm(scev, id, &imm)) { + if (reg->type != SCALAR_VALUE) + return false; + } else if (is_reg(scev, id, &leaf)) { + /* Other leaves must denote loop-invariant registers. */ + if (leaf != r && + !(is_reg(scev, header_env->reg2scev[leaf], &ra) && ra == leaf)) + return false; + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (!leaf_reg || !bpf_reg_union_compatible(reg, leaf_reg)) + return false; + } else { + return false; + } + } + return true; +} + +struct bounds { + struct cnum64 r64; + struct cnum32 r32; + u16 base; + u16 step; +}; + +static void log_bounds(struct bpf_verifier_env *env, const struct bounds *bounds) +{ + struct bpf_verifier_log *log = &env->log; + + bpf_verbose_snum(env, cnum64_smin(bounds->r64)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum64_smax(bounds->r64)); + if (bounds->r32.size < U32_MAX) { + bpf_log(log, " (s32 "); + bpf_verbose_snum(env, cnum32_smin(bounds->r32)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum32_smax(bounds->r32)); + bpf_log(log, ")"); + } + bpf_log(log, " step %u", bounds->step); +} + +struct linear_coefs { + u32 base_reg; + s64 slope; + bool zext32; +}; + +static bool is_simple_linear(struct scev *scev, u32 id, struct linear_coefs *out) +{ + u32 base, slope; + + return is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) && + is_reg(scev, base, &out->base_reg) && + is_imm(scev, slope, &out->slope) && + out->slope <= S16_MAX && + out->slope >= S16_MIN && + out->slope != 0; +} + +static struct cnum64 cnum64_zext32(struct cnum32 r32) +{ + return cnum64_from_urange(cnum32_umin(r32), cnum32_umax(r32)); +} + +/* Compute the range an induction variable in `reg` spans over the loop. */ +static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, + struct linear_coefs *coefs, struct bounds *out) +{ + s64 slope = coefs->slope; + s64 slope_abs = slope < 0 ? -slope : slope; + s64 min_val = reg_smin(reg); + s64 max_val = reg_smax(reg); + struct cnum64 r64; + struct cnum32 r32; + s64 total_change; + bool overflow; + + overflow = check_mul_overflow(slope, (s64)max_header_count - 1, &total_change) || + (slope > 0 && check_add_overflow(max_val, total_change, &max_val)) || + (slope < 0 && check_add_overflow(min_val, total_change, &min_val)); + r64 = overflow ? CNUM64_UNBOUNDED : cnum64_from_srange(min_val, max_val); + r32 = CNUM32_UNBOUNDED; + if (coefs->zext32) { + r32 = cnum32_from_cnum64(r64); + r64 = cnum64_zext32(r32); + /* + * 32-bit iteration overflow is equivalent to subtracting N * 2**32: + * (base + slope * k) % slope ≠ (base + slope * k - N * 2**32) % slope. + */ + overflow |= cnum32_contains(r32, 0) && cnum32_contains(r32, U32_MAX); + /* + * The entry and zero-extended values must agree on how they define `base`, + * otherwise `base + slope * k` would define different sets of points between + * the two. + */ + overflow |= imod(reg_smin(reg), slope_abs) != + imod(cnum64_smin(r64), slope_abs); + /* + * Full range is a union of a reg's value at loop entry, + * plus a range produced by 32-bit iteration. + */ + r64 = cnum64_union(reg->r64, r64); + r32 = cnum32_union(reg->r32, r32); + } + /* + * If the entry value is a single point the value set is 'v + slope * k', + * so the step is |slope|. Otherwise, only the power-of-two alignment + * shared by the entry value and the slope. + */ + if (cnum64_is_const(reg->r64) && !overflow) { + out->step = slope_abs; + out->base = imod(reg_smin(reg), out->step); + } else { + out->step = 1u << min_t(u32, tnum_alignment(reg->var_off), __ffs(slope_abs)); + out->base = 0; + } + out->r64 = r64; + out->r32 = r32; +} + +static int mark_expr_precise(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 id, + unsigned long *precise_regs) +{ + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + u32 r, order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev precision marking exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || !is_reg(scev, id, &r)) + continue; + /* Synthetic values have no spilled register to mark precise. */ + reg = scev_reg_for_linear(st, r); + if (reg && reg->type == SCALAR_VALUE) + __set_bit(r, precise_regs); + } + return 0; +} + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct env *header_env; + struct linear_latch latch, best; + struct linear_coefs coefs; + struct bpf_backedge *backedge; + struct bpf_loop_iters tmp; + struct bpf_loop *loop; + int insn_idx = st->insn_idx; + bool found = false; + int linear_latch; + int latch_idx; + int err, j; + u32 r; + + if (!env->allow_uninit_stack) + return 0; + + /* + * If insn_idx is a loop header for a reducible loop with a single backedge. + * loop is NULL for secondary entries to irreducible loops. + */ + loop = aux[insn_idx].loop; + if (!loop || loop->irreducible || loop->backedges_cnt != 1 || loop->backedges_overflow || + loop->exits_overflow) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, unsupported loop:%s%s%s\n", insn_idx, + !loop || loop->irreducible ? " irreducible" : "", + loop && loop->backedges_cnt > 1 ? " multiple backedges" : "", + loop && loop->exits_overflow ? " too many exits" : ""); + return 0; + } + + /* If this backedge has a latch */ + backedge = &loop->backedges[0]; + if (!backedge->latches_cnt) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, latch not identified\n", insn_idx); + return 0; + } + + /* + * Every condition dominating the backedge bounds the number of + * iterations from above, keep the smallest bound. + */ + for (j = 0; j < backedge->latches_cnt; j++) { + latch_idx = backedge->latches[j]; + linear_latch = match_linear_latch(env, insn_idx, latch_idx, &latch); + if (linear_latch < 0) + return linear_latch; + if (!linear_latch) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, non-linear %s at %d\n", + insn_idx, j ? "exit" : "latch", latch_idx); + continue; + } + if (!compute_max_iters(env, cur_func, &latch, &tmp)) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, + "loop header at %d, can't compute iterations count at %d\n", + insn_idx, latch_idx); + continue; + } + if (!found || tmp.max_header_count < iters->max_header_count) { + iters->max_header_count = tmp.max_header_count; + best = latch; + } + iters->min_header_count = found ? min(iters->min_header_count, tmp.min_header_count) + : tmp.min_header_count; + found = true; + } + if (!found) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, can't compute iterations count\n", insn_idx); + return 0; + } + latch = best; + + if (iters->max_header_count == 0) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, 0 iterations count\n", insn_idx); + return 0; + } + + if (iters->max_header_count == U32_MAX) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, inf iterations count\n", insn_idx); + return 0; + } + + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, header_count is ", insn_idx); + if (iters->min_header_count == iters->max_header_count) + bpf_log(log, "%u ", iters->max_header_count); + else + bpf_log(log, "[%u..%u] ", iters->min_header_count, iters->max_header_count); + bpf_log(log, "\n"); + } + + err = bpf_live_stack_query_init(env, st); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + struct bpf_reg_state *reg; + u32 ra, r_scev, r_expr; + bool spill_base = false; + + if (!scev_reg_alive(env, st, r)) + continue; + /* + * A heuristic: + * Ignore SCEVs computed for unbound values and treat those as widened. + * This is poor man's modelling of e.g. bpf_probe_read_kernel() effect + * on stack, when called from a subprogram. Currently this would produce + * an unknown ('?') SCEV as a summary of a subprogram call effects. + * Ideally, this should be computed as an OPAQUE value. + * However, that would require tracking STACK_MISC writes in the + * liveness analysis. Not implemented yet. + */ + if (is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_scev = header_env->reg2scev[r]; + reg = scev_reg_for_linear(cur_func, r); + /* If SCEV for r is (linear <reg> <slope>) */ + if (reg && is_simple_linear(scev, r_scev, &coefs) && is_widenable_reg_type(reg)) { + /* Spills at varying offsets lose precision */ + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } + continue; + } + + /* rA = rA, loop does not change this reg */ + if (is_reg(scev, r_scev, &ra) && r == ra) + continue; + + /* (any 1 (any 2 (any 3 4))) */ + if (is_widenable_any(env, cur_func, header_env, r, r_scev)) { + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } + continue; + } +cant_widen: + if (log->level & BPF_LOG_LEVEL2) { + r_expr = header_env->reg2expr[r]; + bpf_log(log, "loop header at %d, can't widen ", insn_idx); + log_reg(env, r); + bpf_log(log, ", expr is "); + log_expr(env, r_expr); + if (spill_base) + bpf_log(log, ", requires exact stack-offset tracking"); + bpf_log(log, "\n"); + } + return 0; + } + + bitmap_zero(iters->precise_regs, REGS_NUM); + err = mark_expr_precise(env, cur_func, latch.base_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.step_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.bound_expr, iters->precise_regs); + if (err) + return err; + + return 1; +} + +/* bpf_compute_loop_iters() checked that every leaf can be unioned into acc. */ +static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct bpf_reg_state *acc, u32 id) +{ + struct bpf_reg_state *tmp = &env->fake_reg[0]; + const struct bpf_reg_state *leaf_reg; + struct scev *scev = env->scev; + u32 l, r, leaf, order; + s64 imm; + int err; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev ANY union exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || is_any(scev, id, &l, &r)) + continue; + if (is_imm(scev, id, &imm)) { + bpf_mark_reg_known_scalar(tmp, imm); + leaf_reg = tmp; + } else if (is_reg(scev, id, &leaf)) { + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (verifier_bug_if(!leaf_reg, env, "unreadable SCEV ANY leaf %u", leaf)) + return -EFAULT; + } else { + verifier_bug(env, "scev ANY union has an unsupported leaf"); + return -EFAULT; + } + err = bpf_reg_union(env, acc, leaf_reg); + if (err) + return err; + } + return 0; +} + +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_func_state *entry_func = loop_entry->frame[loop_entry->curframe]; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + struct env *header_env; + struct linear_coefs coefs; + struct bounds bounds; + u32 r, r_expr, a, b; + int insn_idx = st->insn_idx; + int err; + + for_each_set_bit(r, iters->precise_regs, REGS_NUM) { + if (r < MAX_BPF_REG) + bpf_bt_set_frame_reg(&env->bt, st->curframe, r); + else + bpf_bt_set_frame_slot(&env->bt, st->curframe, r - MAX_BPF_REG); + } + err = bpf_mark_chain_precision(env, st, -1, NULL); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + if (!scev_reg_alive(env, st, r) || is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_expr = header_env->reg2scev[r]; + /* If SCEV for r is (linear <reg> <slope>)*/ + if (is_simple_linear(scev, r_expr, &coefs) && coefs.base_reg == r) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + linear_bounds(reg, iters->max_header_count, &coefs, &bounds); + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + log_bounds(env, &bounds); + bpf_log(log, "\n"); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, + bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } else if (is_any(scev, r_expr, &a, &b)) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + err = union_any_reg(env, entry_func, reg, r_expr); + if (err) + return err; + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + bpf_verbose_snum(env, reg_smin(reg)); + bpf_log(log, ".."); + bpf_verbose_snum(env, reg_smax(reg)); + bpf_log(log, " step %u\n", reg->step); + } + scratch_widened_reg_id(env, reg); + mark_scev_reg_scratched(env, r); + } + } + return 1; +} + +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur_func_state, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count) +{ + struct bpf_func_state *entry_st = entry_state->frame[entry_state->curframe]; + struct bpf_verifier_log *log = &env->log; + struct bpf_reg_state *reg, *entry_reg; + struct scev *scev = env->scev; + struct linear_coefs coefs; + struct env *header_env; + struct bounds bounds; + u32 r; + int err; + + if (entry_state->curframe != cur_func_state->frameno) { + verifier_bug(env, "clamping registers for a wrong frame: %d vs %d\n", + entry_state->curframe, cur_func_state->frameno); + return -EFAULT; + } + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + /* If SCEV for r is (linear <reg> <slope>)*/ + if (!is_simple_linear(scev, header_env->reg2scev[r], &coefs) || + coefs.base_reg != r) + continue; + + entry_reg = scev_reg_for_linear(entry_st, r); + reg = scev_reg_for_linear(cur_func_state, r); + if (!entry_reg || !reg || entry_reg->type == NOT_INIT || reg->type == NOT_INIT || + is_unbound_scev_reg(env, entry_st, r)) + continue; + + linear_bounds(entry_reg, max_header_count, &coefs, &bounds); + bounds.r64 = cnum64_intersect(reg->r64, bounds.r64); + bounds.r32 = cnum32_intersect(reg->r32, bounds.r32); + if (cnum64_is_empty(bounds.r64)) { + verifier_bug(env, "scev clamp produced empty range for r%d", r); + return -EFAULT; + } + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, clamping ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + log_bounds(env, &bounds); + bpf_log(log, "\n"); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, + bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } + return 0; +}
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 012b825..d7df663 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c
@@ -302,8 +302,30 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s static bool range_within(const struct bpf_reg_state *old, const struct bpf_reg_state *cur) { - return cnum64_is_subset(old->r64, cur->r64) && - cnum32_is_subset(old->r32, cur->r32); + if (!cnum64_is_subset(old->r64, cur->r64) || + !cnum32_is_subset(old->r32, cur->r32)) + return false; + + if (old->step <= 1) + return true; + + if (cnum64_is_const(cur->r64)) + return imod((s64)cnum64_smin(cur->r64), old->step) == old->base; + + /* + * Both `old` and `cur` define some sets of points. + * Return true, if points defined by `cur` are a subset of points defined by `old`: + * - bounds for `cur` should be within bounds for `old`; + * - cur->step should be dividable by old->step; + * - cur->base should start at integer number of old->step + * steps from old->base. + * + * E.g. the following ranges are compatible: + * - old [0, 10] step 2 + * - new [2, 10] step 4 + */ + return cur->step % old->step == 0 && + (cur->base - old->base) % old->step == 0; } /* If in the old state two registers had the same id, then they need to have @@ -415,14 +437,14 @@ static void __clean_func_state(struct bpf_verifier_env *env, * half_spi 2*i → lower half: slot_type[0..3] (closer to FP) * half_spi 2*i+1 → upper half: slot_type[4..7] (farther from FP) */ - for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(st); i++) { bool lo_live = bpf_stack_slot_alive(env, frame, i * 2); bool hi_live = bpf_stack_slot_alive(env, frame, i * 2 + 1); if (!hi_live || !lo_live) { int start = !lo_live ? 0 : BPF_REG_SIZE / 2; int end = !hi_live ? BPF_REG_SIZE : BPF_REG_SIZE / 2; - u8 stype = st->stack[i].slot_type[7]; + u8 stype = bpf_stack_slot(st, i)->slot_type[7]; /* * Don't clear special slots. @@ -442,7 +464,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, * rejecting as non-scalar register fills. */ if (!hi_live) { - struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; + struct bpf_reg_state *spill = &bpf_stack_slot(st, i)->spilled_ptr; if (lo_live && stype == STACK_SPILL) { if (spill->type != SCALAR_VALUE) @@ -454,7 +476,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (bpf_register_is_null(spill)) continue; for (j = 0; j < 4; j++) { - u8 *t = &st->stack[i].slot_type[j]; + u8 *t = &bpf_stack_slot(st, i)->slot_type[j]; if (*t == STACK_SPILL) *t = STACK_MISC; @@ -463,7 +485,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, spill); } for (j = start; j < end; j++) - st->stack[i].slot_type[j] = STACK_POISON; + bpf_stack_slot(st, i)->slot_type[j] = STACK_POISON; } } } @@ -610,6 +632,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, case PTR_TO_MEM: case PTR_TO_BUF: case PTR_TO_TP_BUFFER: + case PTR_TO_STACK: + case PTR_TO_BTF_ID: /* If the new min/max/var_off satisfy the old ones and * everything else matches, we are OK. */ @@ -643,8 +667,6 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); - case PTR_TO_STACK: - return regs_exact(rold, rcur, idmap); case PTR_TO_ARENA: return true; case PTR_TO_INSN: @@ -709,37 +731,38 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * didn't use them */ for (i = 0; i < old->allocated_stack; i++) { + struct bpf_stack_state *old_slot, *cur_slot; struct bpf_reg_state *old_reg, *cur_reg; int im = i % BPF_REG_SIZE; + u8 old_type; spi = i / BPF_REG_SIZE; + old_slot = bpf_stack_slot(old, spi); + old_type = old_slot->slot_type[im]; + cur_slot = i < cur->allocated_stack ? bpf_stack_slot(cur, spi) : NULL; if (exact == EXACT) { - u8 old_type = old->stack[spi].slot_type[i % BPF_REG_SIZE]; - u8 cur_type = i < cur->allocated_stack ? - cur->stack[spi].slot_type[i % BPF_REG_SIZE] : STACK_INVALID; + u8 cur_type = cur_slot ? cur_slot->slot_type[im] : STACK_INVALID; /* STACK_INVALID and STACK_POISON are equivalent for pruning */ if (old_type == STACK_POISON) old_type = STACK_INVALID; if (cur_type == STACK_POISON) cur_type = STACK_INVALID; - if (i >= cur->allocated_stack || old_type != cur_type) + if (!cur_slot || old_type != cur_type) return false; } - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID || - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_POISON) + if (old_type == STACK_INVALID || old_type == STACK_POISON) continue; - if (env->allow_uninit_stack && - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) + if (env->allow_uninit_stack && old_type == STACK_MISC) continue; /* explored stack has more populated slots than current stack * and these slots were used */ - if (i >= cur->allocated_stack) + if (!cur_slot) return false; /* @@ -749,8 +772,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * regsafe() to ensure scalar ids are compared. */ if (im == 0 || im == 4) { - old_reg = scalar_reg_for_stack(env, &old->stack[spi], im); - cur_reg = scalar_reg_for_stack(env, &cur->stack[spi], im); + old_reg = scalar_reg_for_stack(env, old_slot, im); + cur_reg = scalar_reg_for_stack(env, cur_slot, im); if (old_reg && cur_reg) { if (!regsafe(env, old_reg, cur_reg, idmap, exact)) return false; @@ -763,21 +786,19 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * it will be safe with zero-initialized stack. * The opposite is not true */ - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && - cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) + if (old_type == STACK_MISC && cur_slot->slot_type[im] == STACK_ZERO) continue; - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != - cur->stack[spi].slot_type[i % BPF_REG_SIZE]) + if (old_type != cur_slot->slot_type[im]) /* Ex: old explored (safe) state has STACK_SPILL in * this stack slot, but current has STACK_MISC -> * this verifier states are not equivalent, * return false to continue verification of this path */ return false; - if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) + if (im != BPF_REG_SIZE - 1) continue; /* Both old and cur are having same slot_type */ - switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { + switch (old_type) { case STACK_SPILL: /* when explored and current stack slot are both storing * spilled registers, check that stored pointers types @@ -789,13 +810,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * such verifier states are not equivalent. * return false to continue verification of this path */ - if (!regsafe(env, &old->stack[spi].spilled_ptr, - &cur->stack[spi].spilled_ptr, idmap, exact)) + if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr, + idmap, exact)) return false; break; case STACK_DYNPTR: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (old_reg->dynptr.type != cur_reg->dynptr.type || old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || !check_ids(old_reg->id, cur_reg->id, idmap) || @@ -803,8 +824,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_ITER: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; /* iter.depth is not compared between states as it * doesn't matter for correctness and would otherwise * prevent convergence; we maintain it only to prevent @@ -820,8 +841,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_IRQ_FLAG: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (!check_ids(old_reg->id, cur_reg->id, idmap) || old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class) return false; @@ -915,6 +936,27 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c return true; } +static bool loop_stack_safe(struct bpf_verifier_env *env, struct bpf_func_state *old, + struct bpf_func_state *cur) +{ + struct loop_stack_entry *old_stack = old->loop_stack; + struct loop_stack_entry *cur_stack = cur->loop_stack; + u32 i; + + if (old->loop_stack_cnt != cur->loop_stack_cnt) + return false; + + for (i = 0; i < old->loop_stack_cnt; i++) { + if (old_stack[i].loop_id == cur_stack[i].loop_id && + old_stack[i].max_header_count == cur_stack[i].max_header_count && + old_stack[i].terminates == cur_stack[i].terminates) + continue; + return false; + } + + return true; +} + /* compare two verifier states * * all states stored in state_list are known to be valid, since @@ -953,6 +995,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!old->no_stack_arg_load && cur->no_stack_arg_load) return false; + if (old->in_rcu_callback_fn != cur->in_rcu_callback_fn) + return false; + for (i = 0; i < MAX_BPF_REG; i++) if (((1 << i) & live_regs) && !regsafe(env, &old->regs[i], &cur->regs[i], @@ -965,6 +1010,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact)) return false; + if (!loop_stack_safe(env, old, cur)) + return false; + return true; } @@ -998,6 +1046,9 @@ static bool states_equal(struct bpf_verifier_env *env, if (old->in_sleepable != cur->in_sleepable) return false; + if (old->unwound != cur->unwound) + return false; + if (!refsafe(old, cur, &env->idmap_scratch)) return false; @@ -1011,6 +1062,7 @@ static bool states_equal(struct bpf_verifier_env *env, if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact)) return false; } + return true; } @@ -1045,10 +1097,12 @@ static int propagate_precision(struct bpf_verifier_env *env, first = false; } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (!bpf_is_spilled_reg(&state->stack[i])) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (!bpf_is_spilled_reg(ss)) continue; - state_reg = &state->stack[i].spilled_ptr; + state_reg = &ss->spilled_ptr; if (state_reg->type != SCALAR_VALUE || !state_reg->precise) continue; @@ -1123,7 +1177,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, precise))) + offsetofend(struct bpf_reg_state, step))) return false; return true; } @@ -1194,15 +1248,17 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf for (fr = old->curframe; fr >= 0; fr--) { state = old->frame[fr]; - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (state->stack[i].slot_type[0] != STACK_ITER) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (ss->slot_type[0] != STACK_ITER) continue; - slot = &state->stack[i].spilled_ptr; + slot = &ss->spilled_ptr; if (slot->iter.state != BPF_ITER_STATE_ACTIVE) continue; - cur_slot = &cur->frame[fr]->stack[i].spilled_ptr; + cur_slot = &bpf_stack_slot(cur->frame[fr], i)->spilled_ptr; if (cur_slot->iter.depth != slot->iter.depth) return true; } @@ -1224,10 +1280,12 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ continue; reg->precise = false; } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE) continue; reg->precise = false; @@ -1235,16 +1293,92 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ } } +int bpf_split_cur_state(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state, *new; + struct bpf_verifier_state_list *new_sl; + struct list_head *head; + int insn_idx = cur->insn_idx; + int err; + + head = bpf_explored_state(env, insn_idx); + new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); + if (!new_sl) + return -ENOMEM; + env->total_states++; + env->explored_states_size++; + update_peak_states(env); + env->prev_jmps_processed = env->jmps_processed; + env->prev_insn_processed = env->insn_processed; + + /* forget precise markings we inherited, see __mark_chain_precision */ + if (env->bpf_capable) + mark_all_scalars_imprecise(env, cur); + + bpf_clear_singular_ids(env, cur); + + /* add new state to the head of linked list */ + new = &new_sl->state; + err = bpf_copy_verifier_state(new, cur); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + new->insn_idx = insn_idx; + verifier_bug_if(new->branches != 1, env, + "%s:branches_to_explore=%d insn %d", + __func__, new->branches, insn_idx); + err = maybe_enter_scc(env, new); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + + cur->parent = new; + cur->last_insn_idx = -1; + cur->first_insn_idx = insn_idx; + cur->dfs_depth = new->dfs_depth + 1; + bpf_clear_jmp_history(cur); + list_add(&new_sl->node, head); + return 0; +} + +/* Force a checkpoint upon reaching a loop header for a terminating loop. */ +static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur->frame[cur->curframe]; + struct loop_stack_entry *top; + + if (bpf_loop_at_index(env, insn_idx) != insn_idx || !frame->loop_stack_cnt) + return false; + top = &frame->loop_stack[frame->loop_stack_cnt - 1]; + return top->loop_id == insn_idx && top->terminates; +} + +static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame = st->frame[st->curframe]; + + if (frame->loop_stack_cnt == 0) + return NULL; + return &frame->loop_stack[frame->loop_stack_cnt - 1]; +} + int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) { - struct bpf_verifier_state_list *new_sl; + struct loop_stack_entry *old_loop, *cur_loop; struct bpf_verifier_state_list *sl; - struct bpf_verifier_state *cur = env->cur_state, *new; - bool force_new_state, add_new_state, loop; + struct bpf_verifier_state *cur = env->cur_state; + bool force_new_state, add_new_state, loop, loop_checkpoint; int n, err, states_cnt = 0; struct list_head *pos, *tmp, *head; + loop_checkpoint = need_loop_checkpoint(env, insn_idx); force_new_state = env->test_state_freq || bpf_is_force_checkpoint(env, insn_idx) || + loop_checkpoint || /* Avoid accumulating infinitely long jmp history */ cur->jmp_history_cnt > 40; @@ -1275,10 +1409,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) continue; if (sl->state.branches) { - struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; + struct bpf_func_state *old_top_frame = sl->state.frame[0]; - if (frame->in_async_callback_fn && - frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { + if (old_top_frame->in_async_callback_fn && + old_top_frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) { /* Different async_entry_cnt means that the verifier is * processing another entry into async callback. * Seeing the same state is not an indication of infinite @@ -1330,8 +1464,9 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ if (is_iter_next_insn(env, insn_idx)) { if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { - struct bpf_func_state *cur_frame; + struct bpf_func_state *cur_frame, *iter_frame, *old_frame; struct bpf_reg_state *iter_state, *iter_reg; + struct bpf_reg_state *old_iter; int spi; cur_frame = cur->frame[cur->curframe]; @@ -1344,8 +1479,18 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * no need for extra (re-)validations */ spi = bpf_get_spi(iter_reg->var_off.value); - iter_state = &bpf_func(env, iter_reg)->stack[spi].spilled_ptr; - if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) { + iter_frame = bpf_func(env, iter_reg); + iter_state = &bpf_stack_slot(iter_frame, spi)->spilled_ptr; + old_frame = sl->state.frame[iter_reg->frameno]; + old_iter = &bpf_stack_slot(old_frame, spi)->spilled_ptr; + /* + * states_equal() matches ids through idmap. + * The loop converged only if it's the same + * iterator. The one that was destroyed and + * created again starts from the beginning. + */ + if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE && + iter_state->id == old_iter->id) { loop = true; goto hit; } @@ -1366,6 +1511,33 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) } goto skip_inf_loop_check; } + /* + * If old state belongs to a control flow loop that we know terminates, + * it should be safe to prune current state. However, account for the + * following situation: + * + * for (;;) { + * for (i = 0; i < 10; i++) + * ... + * } + * + * Here a backedge to a non-terminating outer loop is dominated by an exit + * from the inner loop. Pruning at a re-entry to the inner loop therefore + * would hide the fact that outer loop is non-terminating. + * Hence, only allow pruning states within the same entry state. + */ + old_loop = current_loop(&sl->state); + cur_loop = current_loop(cur); + if (old_loop && cur_loop && + old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state) { + if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + loop = true; + goto hit; + } + goto skip_inf_loop_check; + } + /* attempt to detect infinite loop to avoid unnecessary doomed work */ if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur, EXACT) && @@ -1412,7 +1584,7 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ err = 0; if (bpf_is_jmp_point(env, env->insn_idx)) - err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, cur, 0, 0, 0, NULL, 0); err = err ? : propagate_precision(env, &sl->state, cur, NULL); if (err) return err; @@ -1524,7 +1696,8 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * Use bigger 'n' for checkpoints because evicting checkpoint states * too early would hinder iterator convergence. */ - n = bpf_is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; + n = (bpf_is_force_checkpoint(env, insn_idx) || loop_checkpoint) && + sl->state.branches > 0 ? 64 : 3; if (sl->miss_cnt > sl->hit_cnt * n + n) { /* the state is unlikely to be useful. Remove it to * speed up verification @@ -1556,44 +1729,5 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * When looping the sl->state.branches will be > 0 and this state * will not be considered for equivalence until branches == 0. */ - new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); - if (!new_sl) - return -ENOMEM; - env->total_states++; - env->explored_states_size++; - update_peak_states(env); - env->prev_jmps_processed = env->jmps_processed; - env->prev_insn_processed = env->insn_processed; - - /* forget precise markings we inherited, see __mark_chain_precision */ - if (env->bpf_capable) - mark_all_scalars_imprecise(env, cur); - - bpf_clear_singular_ids(env, cur); - - /* add new state to the head of linked list */ - new = &new_sl->state; - err = bpf_copy_verifier_state(new, cur); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - new->insn_idx = insn_idx; - verifier_bug_if(new->branches != 1, env, - "%s:branches_to_explore=%d insn %d", - __func__, new->branches, insn_idx); - err = maybe_enter_scc(env, new); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - - cur->parent = new; - cur->first_insn_idx = insn_idx; - cur->dfs_depth = new->dfs_depth + 1; - bpf_clear_jmp_history(cur); - list_add(&new_sl->node, head); - return 0; + return bpf_split_cur_state(env); }
diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index be9ce98..83e803e 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c
@@ -2,11 +2,15 @@ /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include <linux/bpf.h> +#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf_mem_alloc.h> #include <linux/gfp.h> +#include <linux/irq_work.h> #include <linux/memory.h> #include <linux/mutex.h> +#include <linux/poll.h> +#include <linux/refcount.h> static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len) { @@ -22,11 +26,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) size_t alloc_size; /* - * Length denotes the amount of data to be written as part of stream element, - * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can - * accomodate, therefore deny allocations that won't fit into them. + * Length is the payload pushed into the stream, excluding the + * trailing NUL of the bprintf buffer. Reject anything that cannot + * fit without copying that NUL into the stream element. */ - if (len < 0 || len > max_len) + if (len < 0 || len >= max_len) return NULL; alloc_size = offsetof(struct bpf_stream_elem, str[len]); @@ -68,25 +72,82 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) return 0; } -static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) +static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) { - int len = elem->total_len; - atomic_sub(len, &stream->capacity); } +static void bpf_stream_notify(struct irq_work *work) +{ + struct bpf_stream *stream = container_of(work, struct bpf_stream, notify_work); + + /* + * Writers run in arbitrary program contexts, including NMI and regions + * that already hold wait queue or epoll locks. Wake waiters from + * irq_work instead, where taking those locks is safe. + */ + wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM); +} + +static void bpf_stream_queue_notify(struct bpf_stream *stream) +{ + /* + * Record that the work has been used so that teardown only pays for + * irq_work_sync(), which may wait for an RCU grace period, when a + * callback could actually be in flight. + */ + if (!READ_ONCE(stream->notify_used)) + WRITE_ONCE(stream->notify_used, true); + irq_work_queue(&stream->notify_work); +} + +static int bpf_stream_readable_bytes(struct bpf_stream *stream) +{ + return atomic_read_acquire(&stream->readable); +} + +static void bpf_stream_publish(struct bpf_stream *stream, int len) +{ + /* + * Pairs with atomic_read_acquire() in bpf_stream_readable_bytes(). + * + * Notify only when the stream turns from empty to readable. Readers + * block and pollers wait only after finding it empty, and a read never + * takes the count below zero, so the publication that makes it positive + * is the one they wait for. Publishing into a stream that already holds + * data would only raise an interrupt for waiters that are being woken + * already, or for nobody at all. + */ + if (atomic_add_return_release(len, &stream->readable) == len) + bpf_stream_queue_notify(stream); +} + static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) { - int ret = bpf_stream_consume_capacity(stream, len); + int ret; - return ret ?: __bpf_stream_push_str(&stream->log, str, len); + /* Nothing to publish; do not allocate an element for it. */ + if (!len) + return 0; + + ret = bpf_stream_consume_capacity(stream, len); + if (ret) + return ret; + + ret = __bpf_stream_push_str(&stream->log, str, len); + if (ret) + bpf_stream_release_capacity(stream, len); + else + bpf_stream_publish(stream, len); + + return ret; } static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) { if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR) return NULL; - return &aux->stream[stream_id - 1]; + return aux->stream[stream_id - 1]; } static void bpf_stream_free_elem(struct bpf_stream_elem *elem) @@ -154,14 +215,17 @@ static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len) static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) { - int rem_len = len, cons_len, ret = 0; + int read_len, rem_len, cons_len, ret = 0; struct bpf_stream_elem *elem = NULL; struct llist_node *node; mutex_lock(&stream->lock); + read_len = min(len, bpf_stream_readable_bytes(stream)); + rem_len = read_len; while (rem_len) { - int pos = len - rem_len; + int pos = read_len - rem_len; + int chunk, n; bool cont; node = bpf_stream_backlog_peek(stream); @@ -175,37 +239,146 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) cons_len = elem->consumed_len; cont = bpf_stream_consume_elem(elem, &rem_len) == false; + chunk = elem->consumed_len - cons_len; - ret = copy_to_user(buf + pos, elem->str + cons_len, - elem->consumed_len - cons_len); - /* Restore in case of error. */ - if (ret) { - ret = -EFAULT; - elem->consumed_len = cons_len; + n = copy_to_user(buf + pos, elem->str + cons_len, chunk); + if (n) { + /* Keep any successfully copied bytes; -EFAULT only if none. */ + elem->consumed_len -= n; + rem_len += n; + ret = (read_len == rem_len) ? -EFAULT : 0; break; } if (cont) continue; bpf_stream_backlog_pop(stream); - bpf_stream_release_capacity(stream, elem); + bpf_stream_release_capacity(stream, elem->total_len); bpf_stream_free_elem(elem); } + atomic_sub(read_len - rem_len, &stream->readable); mutex_unlock(&stream->lock); - return ret ? ret : len - rem_len; + return ret ? ret : read_len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } +static bool bpf_stream_has_data(struct bpf_stream *stream) +{ + return bpf_stream_readable_bytes(stream) > 0; +} + +static void bpf_stream_put(struct bpf_stream *stream) +{ + if (refcount_dec_and_test(&stream->refcnt)) { + struct llist_node *list; + + /* Only a stream that ever queued its work can have a callback in flight. */ + if (READ_ONCE(stream->notify_used)) + irq_work_sync(&stream->notify_work); + list = llist_del_all(&stream->log); + bpf_stream_free_list(list); + bpf_stream_free_list(stream->backlog_head); + mutex_destroy(&stream->lock); + kfree(stream); + } +} + +static int bpf_stream_release(struct inode *inode, struct file *file) +{ + bpf_stream_put(file->private_data); + return 0; +} + +static ssize_t bpf_stream_file_read(struct file *file, char __user *buf, size_t len, + loff_t *ppos) +{ + struct bpf_stream *stream = file->private_data; + bool dead; + int ret; + + if (!len) + return 0; + + for (;;) { + /* + * Sample teardown state before looking for data. Nothing is + * published once the program is gone, so finding the stream + * empty after observing dead means EOF. The opposite order could + * report EOF while data published just before teardown is still + * buffered. + */ + dead = smp_load_acquire(&stream->dead); + ret = bpf_stream_read(stream, buf, len); + if (ret) + return ret; + if (dead) + return 0; + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(stream->waitq, + bpf_stream_has_data(stream) || + READ_ONCE(stream->dead)); + if (ret) + return ret; + } +} + +static __poll_t bpf_stream_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_stream *stream = file->private_data; + __poll_t events = 0; + + /* + * poll_wait() only registers the wait queue callback. Register before + * checking persistent state so a concurrent publication or teardown is + * observed either by the callback or by the checks below. + */ + poll_wait(file, &stream->waitq, pts); + if (bpf_stream_has_data(stream)) + events |= EPOLLIN | EPOLLRDNORM; + if (READ_ONCE(stream->dead)) + events |= EPOLLHUP; + return events; +} + +static const struct file_operations bpf_stream_fops = { + .release = bpf_stream_release, + .read = bpf_stream_file_read, + .poll = bpf_stream_poll, +}; + +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags) +{ + struct bpf_stream *stream; + int fd_flags = O_RDONLY | O_CLOEXEC; + int fd; + + stream = bpf_stream_get(stream_id, prog->aux); + if (!stream) + return -ENOENT; + if (flags & BPF_F_STREAM_NONBLOCK) + fd_flags |= O_NONBLOCK; + + refcount_inc(&stream->refcnt); + fd = anon_inode_getfd("bpf-stream", &bpf_stream_fops, stream, fd_flags); + if (fd < 0) + bpf_stream_put(stream); + return fd; +} + __bpf_kfunc_start_defs(); /* @@ -238,6 +411,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo return ret; ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); + /* Truncation: reject before capacity charge (not -ENOMEM). */ + if (ret >= MAX_BPRINTF_BUF) { + bpf_bprintf_cleanup(&data); + return -E2BIG; + } /* Exclude NULL byte during push. */ ret = bpf_stream_push_str(stream, data.buf, ret); bpf_bprintf_cleanup(&data); @@ -268,28 +446,47 @@ __bpf_kfunc_end_defs(); /* Added kfunc to common_btf_ids */ -void bpf_prog_stream_init(struct bpf_prog *prog) +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags) { int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - atomic_set(&prog->aux->stream[i].capacity, 0); - init_llist_head(&prog->aux->stream[i].log); - mutex_init(&prog->aux->stream[i].lock); - prog->aux->stream[i].backlog_head = NULL; - prog->aux->stream[i].backlog_tail = NULL; + struct bpf_stream *stream; + + /* On failure, bpf_prog_stream_free() releases the streams allocated so far. */ + stream = kzalloc_obj(*stream, + bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); + if (!stream) + return -ENOMEM; + + refcount_set(&stream->refcnt, 1); + init_llist_head(&stream->log); + mutex_init(&stream->lock); + init_waitqueue_head(&stream->waitq); + init_irq_work(&stream->notify_work, bpf_stream_notify); + prog->aux->stream[i] = stream; } + return 0; } void bpf_prog_stream_free(struct bpf_prog *prog) { - struct llist_node *list; int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - list = llist_del_all(&prog->aux->stream[i].log); - bpf_stream_free_list(list); - bpf_stream_free_list(prog->aux->stream[i].backlog_head); + struct bpf_stream *stream = prog->aux->stream[i]; + + if (!stream) + continue; + /* + * Pairs with smp_load_acquire() in bpf_stream_file_read(): every + * publication precedes the dead flag, so a reader that observes + * it also observes all buffered data. + */ + smp_store_release(&stream->dead, true); + wake_up_interruptible_poll(&stream->waitq, EPOLLHUP); + bpf_stream_put(stream); + prog->aux->stream[i] = NULL; } } @@ -311,17 +508,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) { struct bpf_bprintf_buffers *buf; va_list args; - int ret; + int len, ret; if (bpf_try_get_buffers(&buf)) return -EBUSY; va_start(args, fmt); - ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); + len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - ss->len += ret; - /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); + /* Exclude NULL byte during push; skip empty output entirely. */ + ret = len ? __bpf_stream_push_str(&ss->log, buf->buf, len) : 0; + if (!ret) + ss->len += len; bpf_put_buffers(); return ret; } @@ -351,6 +549,7 @@ int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, list = tail; } llist_add_batch(head, tail, &stream->log); + bpf_stream_publish(stream, ss->len); return 0; }
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 96217b9..90bc723 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c
@@ -602,38 +602,45 @@ static bool can_alloc_pages(void) !IS_ENABLED(CONFIG_PREEMPT_RT); } -static struct page *__bpf_alloc_page(int nid) +struct page *bpf_alloc_page(int nid, bool sleepable) { - if (!can_alloc_pages()) + if (!sleepable || !can_alloc_pages()) return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0); return alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT - | __GFP_NOWARN, + | __GFP_NOWARN | __GFP_RETRY_MAYFAIL, 0); } -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **pages) +void bpf_free_pages(struct llist_head *pages) { - unsigned long i, j; + struct llist_node *node; + struct page *page, *tmp; + + node = llist_del_all(pages); + llist_for_each_entry_safe(page, tmp, node, pcp_llist) + free_pages_nolock(page, 0); +} + +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable) +{ + unsigned long i; struct page *pg; - int ret = 0; for (i = 0; i < nr_pages; i++) { - pg = __bpf_alloc_page(nid); - - if (pg) { - pages[i] = pg; - continue; - } - for (j = 0; j < i; j++) - free_pages_nolock(pages[j], 0); - ret = -ENOMEM; - break; + pg = bpf_alloc_page(nid, sleepable); + if (!pg) + goto free_pages; + llist_add(&pg->pcp_llist, pages); } - return ret; + return 0; + +free_pages: + bpf_free_pages(pages); + return -ENOMEM; } static int btf_field_cmp(const void *a, const void *b) @@ -687,6 +694,7 @@ void btf_record_free(struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to release */ break; default: @@ -741,6 +749,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to acquire */ break; default: @@ -874,6 +883,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) case BPF_LIST_NODE: case BPF_RB_NODE: case BPF_REFCOUNT: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -954,6 +964,9 @@ void bpf_map_put(struct bpf_map *map) /* bpf_map_free_id() must be called first */ bpf_map_free_id(map); + if (map->ops->map_free_pre_rcu) + map->ops->map_free_pre_rcu(map); + WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); /* RCU tasks trace grace period implies RCU grace period. */ if (READ_ONCE(map->free_after_mult_rcu_gp)) @@ -1277,7 +1290,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, map->record = btf_parse_fields(btf, value_type, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD | BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR | - BPF_TASK_WORK, + BPF_TASK_WORK | BPF_RCU_HEAD, map->value_size); if (!IS_ERR_OR_NULL(map->record)) { int i; @@ -1319,6 +1332,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, goto free_map_tab; } break; + case BPF_RCU_HEAD: + if (map->map_type != BPF_MAP_TYPE_ARRAY) { + ret = -EOPNOTSUPP; + goto free_map_tab; + } + break; case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: @@ -2441,6 +2460,9 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) { struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + btf_put(aux->btf); + kvfree(aux->jited_linfo); + kvfree(aux->linfo); kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); @@ -2457,7 +2479,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) { struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); else call_rcu(rcu, __bpf_prog_put_rcu); @@ -2465,11 +2487,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { - bpf_prog_kallsyms_del_all(prog); - btf_put(prog->aux->btf); module_put(prog->aux->mod); - kvfree(prog->aux->jited_linfo); - kvfree(prog->aux->linfo); kfree(prog->aux->kfunc_tab); kfree(prog->aux->ctx_arg_info); if (prog->aux->attach_btf) @@ -2478,7 +2496,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) if (deferred) { if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); - else if (prog->sleepable) + else if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); @@ -2953,9 +2971,13 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | + BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; + if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) + return -EOPNOTSUPP; + bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { @@ -3085,6 +3107,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; + err = bpf_prog_stream_init(prog, GFP_USER); + if (err) + goto free_prog; + err = -EFAULT; if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), @@ -3824,6 +3850,19 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -EINVAL; goto out_unlock; } + /* + * The size of these contexts depends on what the program is attached + * to, or run with. That is compared with max_ctx_offset of the program, + * and nobody looks at the one of its extensions. + */ + if (prog->type == BPF_PROG_TYPE_EXT && + (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_SYSCALL) && + prog->aux->max_ctx_offset > tgt_prog->aux->max_ctx_offset) { + err = -EINVAL; + goto out_unlock; + } err = bpf_link_prime(&link->link.link, &link_primer); if (err) @@ -3993,19 +4032,19 @@ static int bpf_perf_link_fill_common(const struct perf_event *event, *ulenp = 1; } if (!uname) - return 0; + goto out; if (buf) { err = bpf_copy_to_user(uname, buf, ulen, len); - if (err) - return err; } else { char zero = '\0'; if (put_user(zero, uname)) - return -EFAULT; + err = -EFAULT; } - return 0; +out: + kfree_const(buf); + return err; } #ifdef CONFIG_KPROBE_EVENTS @@ -4062,6 +4101,7 @@ static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event, name, offset, missed, addr, type == BPF_FD_TYPE_KRETPROBE ? "kretprobe" : "kprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4115,6 +4155,7 @@ static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event, name, offset, ref_ctr_offset, type == BPF_FD_TYPE_URETPROBE ? "uretprobe" : "uprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4213,6 +4254,7 @@ static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event, "event_type:\t%s\n" "cookie:\t%llu\n", name, "tracepoint", event->bpf_cookie); + kfree_const(name); } static void bpf_probe_link_show_fdinfo(const struct perf_event *event, @@ -4772,6 +4814,7 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: + case BPF_STRUCT_OPS: return cgroup_bpf_prog_query(attr, uattr, uattr_size); case BPF_LIRC_MODE2: return lirc_prog_query(attr, uattr); @@ -5044,6 +5087,12 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, insns[i].imm = BPF_FUNC_tail_call; /* fall-through */ } + /* + * BPF_JCOND is the same opcode as BPF_CALL_ARGS. The latter is for + * the interpreter, and a prog with unwind_goto is always JITed. + */ + if (prog->jited && bpf_is_unwind_goto(&insns[i])) + continue; if (code == (BPF_JMP | BPF_CALL) || code == (BPF_JMP | BPF_CALL_ARGS)) { /* Restore the legacy xlated dump layout. */ @@ -5420,7 +5469,11 @@ static int bpf_map_get_info_by_fd(struct file *file, info_len = min_t(u32, sizeof(info), info_len); memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_len)) + if (info_len >= offsetofend(struct bpf_map_info, hash) && + get_user(info.hash, &uinfo->hash)) + return -EFAULT; + if (info_len >= offsetofend(struct bpf_map_info, hash_size) && + get_user(info.hash_size, &uinfo->hash_size)) return -EFAULT; info.type = map->map_type; @@ -5735,11 +5788,13 @@ static int bpf_task_fd_query(const union bpf_attr *attr, err = bpf_get_perf_event_info(event, &prog_id, &fd_type, &buf, &probe_offset, &probe_addr, NULL); - if (!err) + if (!err) { err = bpf_task_fd_query_copy(attr, uattr, prog_id, fd_type, buf, probe_offset, probe_addr); + kfree_const(buf); + } goto put_file; } @@ -6220,7 +6275,7 @@ static int bpf_prog_bind_map(union bpf_attr *attr) /* The bpf program will not access the bpf map, but for the sake of * simplicity, increase sleepable_refcnt for sleepable program as well. */ - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); @@ -6320,6 +6375,28 @@ static int prog_assoc_struct_ops(union bpf_attr *attr) return ret; } +#define BPF_PROG_STREAM_OPEN_LAST_FIELD prog_stream_open.flags + +static int prog_stream_open(union bpf_attr *attr) +{ + struct bpf_prog *prog; + u32 flags = attr->prog_stream_open.flags; + int ret; + + if (CHECK_ATTR(BPF_PROG_STREAM_OPEN)) + return -EINVAL; + if (flags & ~BPF_F_STREAM_NONBLOCK) + return -EINVAL; + + prog = bpf_prog_get(attr->prog_stream_open.prog_fd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + ret = bpf_prog_stream_new_fd(prog, attr->prog_stream_open.stream_id, flags); + bpf_prog_put(prog); + return ret; +} + static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, bpfptr_t uattr_common, unsigned int size_common) { @@ -6492,6 +6569,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, case BPF_PROG_ASSOC_STRUCT_OPS: err = prog_assoc_struct_ops(&attr); break; + case BPF_PROG_STREAM_OPEN: + err = prog_stream_open(&attr); + break; default: err = -EINVAL; break;
diff --git a/kernel/bpf/sysfs_btf.c b/kernel/bpf/sysfs_btf.c index 9cbe15c..556fe92 100644 --- a/kernel/bpf/sysfs_btf.c +++ b/kernel/bpf/sysfs_btf.c
@@ -6,8 +6,10 @@ #include <linux/module.h> #include <linux/kobject.h> #include <linux/init.h> +#include <linux/slab.h> #include <linux/sysfs.h> #include <linux/mm.h> +#include <linux/vmalloc.h> #include <linux/io.h> #include <linux/btf.h> @@ -15,16 +17,12 @@ extern char __start_BTF[]; extern char __stop_BTF[]; -static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, - const struct bin_attribute *attr, - struct vm_area_struct *vma) +static int btf_sysfs_mmap_check(void *data, size_t size, struct vm_area_struct *vma) { - unsigned long pages = PAGE_ALIGN(attr->size) >> PAGE_SHIFT; + unsigned long pages = PAGE_ALIGN(size) >> PAGE_SHIFT; size_t vm_size = vma->vm_end - vma->vm_start; - phys_addr_t addr = __pa_symbol(__start_BTF); - unsigned long pfn = addr >> PAGE_SHIFT; - if (attr->private != __start_BTF || !PAGE_ALIGNED(addr)) + if (!data || !PAGE_ALIGNED((unsigned long)data)) return -EINVAL; if (vma->vm_pgoff) @@ -33,24 +31,153 @@ static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_MAYSHARE)) return -EACCES; - if (pfn + pages < pfn) - return -EINVAL; - if ((vm_size >> PAGE_SHIFT) > pages) return -EINVAL; vm_flags_mod(vma, VM_DONTDUMP, VM_MAYEXEC | VM_MAYWRITE); + return 0; +} + +static int btf_sysfs_mmap_image(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = READ_ONCE(attr->private); + phys_addr_t addr; + unsigned long pfn; + size_t vm_size = vma->vm_end - vma->vm_start; + int err; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + /* Kernel image addresses can lie inside the vmalloc range on arm64. */ + addr = __pa_symbol(data); + pfn = addr >> PAGE_SHIFT; + if (pfn + (PAGE_ALIGN(attr->size) >> PAGE_SHIFT) < pfn) + return -EINVAL; + return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot); } static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = { .attr = { .name = "vmlinux", .mode = 0444, }, .read = sysfs_bin_attr_simple_read, - .mmap = btf_sysfs_vmlinux_mmap, + .mmap = btf_sysfs_mmap_image, }; struct kobject *btf_kobj; +struct btf_sysfs_entry { + struct bin_attribute attr; + char *module_name; +}; + +static void *btf_sysfs_lazy_data(const struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + void *data; + + data = smp_load_acquire(&attr->private); + if (!data) { + request_module("%s", entry->module_name); + data = smp_load_acquire(&attr->private); + } + return data; +} + +static ssize_t btf_sysfs_read_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + void *data = btf_sysfs_lazy_data(attr); + + if (!data) + return -ENODEV; + + return memory_read_from_buffer(buf, count, &off, data, attr->size); +} + +static int btf_sysfs_mmap_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = btf_sysfs_lazy_data(attr); + int err; + + if (!data) + return -ENODEV; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + return remap_vmalloc_range(vma, data, 0); +} + +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name) +{ + struct btf_sysfs_entry *entry; + struct bin_attribute *attr; + int err; + + entry = kzalloc_obj(*entry); + if (!entry) + return ERR_PTR(-ENOMEM); + + attr = &entry->attr; + sysfs_bin_attr_init(attr); + attr->attr.mode = 0444; + attr->size = data_size; + attr->private = data; + attr->read = lazy_module_name ? btf_sysfs_read_lazy : sysfs_bin_attr_simple_read; + if (mmap) + attr->mmap = lazy_module_name ? btf_sysfs_mmap_lazy : btf_sysfs_mmap_image; + attr->attr.name = kstrdup(name, GFP_KERNEL); + if (!attr->attr.name) { + err = -ENOMEM; + goto err_free; + } + if (lazy_module_name) { + entry->module_name = kstrdup(lazy_module_name, GFP_KERNEL); + if (!entry->module_name) { + err = -ENOMEM; + goto err_free; + } + } + + err = sysfs_create_bin_file(btf_kobj, attr); + if (err) { + pr_warn("failed to register [%s] BTF in sysfs: %d\n", name, err); + goto err_free; + } + + return attr; + +err_free: + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); + return ERR_PTR(err); +} + +bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return attr && !cmpxchg(&attr->private, NULL, data); +} + +void sysfs_btf_remove(struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + + sysfs_remove_bin_file(btf_kobj, attr); + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); +} + static int __init btf_vmlinux_init(void) { bin_attr_btf_vmlinux.private = __start_BTF;
diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index c65ba1d..2ee7b3b 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c
@@ -737,7 +737,11 @@ static const struct bpf_iter_seq_info task_vma_seq_info = { static struct bpf_iter_reg task_vma_reg_info = { .target = "task_vma", .attach_target = bpf_iter_attach_task, - .feature = BPF_ITER_RESCHED, + /* + * ->show() runs with the mmap_lock of the iterated mm held, which a + * sleepable program takes again when it faults on user memory. + */ + .feature = BPF_ITER_RESCHED | BPF_ITER_NOSLEEP, .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__task_vma, task), @@ -806,8 +810,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, vma = find_vma(mm, start); if (vma && vma->vm_start <= start && vma->vm_end > start) { + /* + * The callback must not sleep while mmap_lock is held, and the + * verifier checks it as an RCU read-side critical section. + */ + rcu_read_lock(); callback_fn((u64)(long)task, (u64)(long)vma, (u64)(long)callback_ctx, 0, 0); + rcu_read_unlock(); ret = 0; } bpf_mmap_unlock_mm(work, mm);
diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index ec9c310..1f620fa 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c
@@ -317,3 +317,14 @@ u64 tnum_step(struct tnum t, u64 z) inc = (filled + 1) & t.mask; return t.value | inc; } + +/* + * Return the number of trailing bits known to be zero in a tnum. + * Return 64 for a tnum representing zero. + */ +u32 tnum_alignment(struct tnum a) +{ + u64 v = a.value | a.mask; + + return v ? __ffs64(v) : 64; +}
diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index da915a4..aa42243 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c
@@ -169,8 +169,11 @@ int bpf_token_create(union bpf_attr *attr) FD_PREPARE(fdf, O_CLOEXEC, alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops)); - if (fdf->fd < 0) + if (fdf->fd < 0) { + /* no file took over the inode reference */ + iput(inode); return fdf->fd; + } token = kzalloc_obj(*token, GFP_USER); if (!token)
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index bf4cb3d..21d4608 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c
@@ -475,18 +475,11 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im void *data __maybe_unused) { void *new_addr = im->image; - void *ip = tr->func.addr; - unsigned long faddr; int ret; - faddr = ftrace_location((unsigned long)ip); - if (faddr) { + if (tr->func.ftrace_managed) { if (!tr->fops) return -ENOTSUPP; - tr->func.ftrace_managed = true; - } - - if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); @@ -859,7 +852,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) +static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; @@ -873,7 +866,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) */ return -EBUSY; - aux->is_extended = true; + aux->freplace_link_cnt++; return 0; } @@ -1023,13 +1016,20 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); + err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; + err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, + BPF_MOD_JUMP, NULL, + node->link->prog->bpf_func); + if (err) { + /* Undo the claim from bpf_freplace_link_tgt_prog(). */ + guard(mutex)(&tgt_prog->aux->ext_mutex); + tgt_prog->aux->freplace_link_cnt--; + return err; + } tr->extension_prog = node->link->prog; - return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - node->link->prog->bpf_func); + return 0; } err = bpf_trampoline_add_prog(tr, node, cnt); if (err) @@ -1069,7 +1069,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; guard(mutex)(&tgt_prog->aux->ext_mutex); - tgt_prog->aux->is_extended = false; + tgt_prog->aux->freplace_link_cnt--; return err; } bpf_trampoline_remove_prog(tr, node); @@ -1279,6 +1279,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; + tr->func.ftrace_managed = tr->ip != 0; out: trampoline_unlock(tr); return tr; @@ -1628,13 +1629,20 @@ static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry * static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + unsigned long ip = tr->ip; + + /* + * Different BTF ids get different trampolines, but they can still + * resolve to the same address (aliases, functions of the same name). + * There's only one call site to attach to. + */ + if (ftrace_lookup_ip(data->reg, ip)) + return -EINVAL; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1643,13 +1651,12 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) { unsigned long addr = (unsigned long) tr->cur_image->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->unreg, data->entry, ip, addr); + ftrace_hash_add(data->unreg, data->entry, tr->ip, addr); tr->cur_image = NULL; return 0; } @@ -1658,13 +1665,12 @@ static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bool lock_direct_mutex, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->modify, data->entry, ip, addr); + ftrace_hash_add(data->modify, data->entry, tr->ip, addr); tr->cur_image = im; return 0; }
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5f87497..5601d20 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c
@@ -5,6 +5,7 @@ */ #include <uapi/linux/btf.h> #include <linux/bpf-cgroup.h> +#include <linux/count_zeros.h> #include <linux/kernel.h> #include <linux/types.h> #include <linux/slab.h> @@ -24,11 +25,13 @@ #include <linux/bpf_lsm.h> #include <linux/security.h> #include <linux/verification.h> +#include <linux/keyctl.h> #include <linux/btf_ids.h> #include <linux/poison.h> #include <linux/module.h> #include <linux/cpumask.h> #include <linux/cnum.h> +#include <linux/gcd.h> #include <linux/bpf_mem_alloc.h> #include <net/xdp.h> #include <linux/trace_events.h> @@ -216,6 +219,7 @@ static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg); static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); static inline bool in_sleepable_context(struct bpf_verifier_env *env); +static bool in_rcu_callback(struct bpf_verifier_env *env); static const char *non_sleepable_context_description(struct bpf_verifier_env *env); static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); @@ -301,6 +305,12 @@ static int arg_idx_from_argno(argno_t a) return arg_from_argno(a) - 1; } +/* Normalize helper register numbers and kfunc argument numbers to ABI slots. */ +static u32 arg_slot_from_argno(argno_t a) +{ + return abs(a.argno) - 1; +} + static const char *btf_type_name(const struct btf *btf, u32 id) { return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); @@ -394,27 +404,69 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} - return btf_type_is_void(type); +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; + + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but btf_check_func_type_match() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + if (!type || btf_type_is_void(type)) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -486,6 +538,7 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) static bool is_sync_callback_calling_kfunc(u32 btf_id); static bool is_async_callback_calling_kfunc(u32 btf_id); +static bool is_call_rcu_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); @@ -528,6 +581,10 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false; + /* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */ + if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm)) + return false; + /* bpf_wq and bpf_task_work callbacks are always sleepable. */ if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) @@ -537,6 +594,19 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn return false; } +/* From here on the maps of the program count it as a sleepable user. */ +static void mark_sleepable_async_cb(struct bpf_verifier_env *env) +{ + int i; + + if (bpf_prog_runs_sleepable(env->prog)) + return; + + env->prog->aux->sleepable_async_cb = true; + for (i = 0; i < env->used_map_cnt; i++) + atomic64_inc(&env->used_maps[i]->sleepable_refcnt); +} + bool bpf_is_may_goto_insn(struct bpf_insn *insn) { return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; @@ -544,16 +614,17 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn) static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) { - int allocated_slots = state->allocated_stack / BPF_REG_SIZE; + int allocated_slots = bpf_stack_nr_slots(state); - /* We need to check that slots between [spi - nr_slots + 1, spi] are - * within [0, allocated_stack). - * - * Please note that the spi grows downwards. For example, a dynptr - * takes the size of two stack slots; the first slot will be at - * spi and the second slot will be at spi - 1. - */ - return spi - nr_slots + 1 >= 0 && spi < allocated_slots; + /* + * We need to check that slots between [spi - nr_slots + 1, spi] are + * within [0, allocated_stack). + * + * Please note that the spi grows downwards. For example, a dynptr + * takes the size of two stack slots; the first slot will be at + * spi and the second slot will be at spi - 1. + */ + return spi - nr_slots + 1 >= 0 && spi < allocated_slots; } static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -668,6 +739,21 @@ static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi); +static int destroy_dynptrs_in_stack_slots(struct bpf_verifier_env *env, + struct bpf_func_state *state, + int first_spi, int last_spi) +{ + int spi, err; + + for (spi = first_spi; spi <= last_spi; spi++) { + err = destroy_if_dynptr_stack_slot(env, state, spi); + if (err) + return err; + } + + return 0; +} + static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type, int insn_idx, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) @@ -681,7 +767,7 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ return spi; /* We cannot assume both spi and spi - 1 belong to the same dynptr, - * hence we need to call destroy_if_dynptr_stack_slot twice for both, + * hence we need to destroy dynptrs in both slots, * to ensure that for the following example: * [d1][d1][d2][d2] * spi 3 2 1 0 @@ -689,16 +775,13 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ * case they do belong to same dynptr, second call won't see slot_type * as STACK_DYNPTR and will simply skip destruction. */ - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - err = destroy_if_dynptr_stack_slot(env, state, spi - 1); + err = destroy_dynptrs_in_stack_slots(env, state, spi - 1, spi); if (err) return err; for (i = 0; i < BPF_REG_SIZE; i++) { - state->stack[spi].slot_type[i] = STACK_DYNPTR; - state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR; } type = arg_to_dynptr_type(arg_type); @@ -731,8 +814,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ parent_id = dynptr->parent_id; } - mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, - &state->stack[spi - 1].spilled_ptr, type, parent_id); + mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr, + &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id); return 0; } @@ -764,7 +847,7 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re * all clones and derived slices. For non-referenced dynptr, only * the dynptr and slices derived from it will be invalidated. */ - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; return release_reference(env, dynptr_type_referenced(reg->dynptr.type) ? reg->parent_id : reg->id); @@ -803,6 +886,7 @@ static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id) static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int err = 0; /* We always ensure that STACK_DYNPTR is never set partially, @@ -810,20 +894,22 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, * different for STACK_SPILL, where it may be only set for * 1 byte, so code has to use is_spilled_reg. */ - if (state->stack[spi].slot_type[0] != STACK_DYNPTR) + if (slot->slot_type[0] != STACK_DYNPTR) return 0; /* Reposition spi to first slot */ - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + if (!slot->spilled_ptr.dynptr.first_slot) { spi = spi + 1; + slot = bpf_stack_slot(state, spi); + } /* * A referenced dynptr can be overwritten only if there is at * least one other dynptr sharing the same virtual ref parent, * ensuring the reference can still be properly released. */ - if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && - dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { + if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) && + dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); bpf_diag_res( env, env->insn_idx, "referenced dynptr overwrite", @@ -833,7 +919,7 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, } /* Invalidate the dynptr and any derived slices */ - err = release_reference(env, state->stack[spi].spilled_ptr.id); + err = release_reference(env, slot->spilled_ptr.id); if (!err) { mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi - 1); @@ -873,6 +959,7 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct bpf_func_state *state = bpf_func(env, reg); + struct bpf_stack_state *slot; int i, spi; /* This already represents first slot of initialized bpf_dynptr. @@ -887,12 +974,13 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re spi = dynptr_get_spi(env, reg); if (spi < 0) return false; - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + slot = bpf_stack_slot(state, spi); + if (!slot->spilled_ptr.dynptr.first_slot) return false; for (i = 0; i < BPF_REG_SIZE; i++) { - if (state->stack[spi].slot_type[i] != STACK_DYNPTR || - state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) + if (slot->slot_type[i] != STACK_DYNPTR || + bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR) return false; } @@ -911,14 +999,23 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct if (spi < 0) return BPF_DYNPTR_TYPE_INVALID; state = bpf_func(env, reg); - return state->stack[spi].spilled_ptr.dynptr.type; + return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type; +} + +static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type) +{ + return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META; } static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { - /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ - if (arg_type == ARG_PTR_TO_DYNPTR) + /* + * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr. + * Test the flags rather than the whole arg_type, which may carry + * unrelated ones such as PTR_MAYBE_NULL. + */ + if (!(arg_type & DYNPTR_TYPE_FLAG_MASK)) return true; return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); @@ -947,7 +1044,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env, return id; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -984,7 +1081,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, return spi; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (i == 0) @@ -1018,7 +1115,7 @@ static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, return false; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); for (j = 0; j < BPF_REG_SIZE; j++) if (slot->slot_type[j] == STACK_ITER) @@ -1039,7 +1136,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s return -EINVAL; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (st->type & PTR_UNTRUSTED) @@ -1081,7 +1178,7 @@ static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, if (id < 0) return id; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -1108,7 +1205,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return spi; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (st->irq.kfunc_class != kfunc_class) { @@ -1177,7 +1274,7 @@ static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bp if (spi < 0) return false; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); for (i = 0; i < BPF_REG_SIZE; i++) if (slot->slot_type[i] == STACK_IRQ_FLAG) @@ -1196,7 +1293,7 @@ static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return -EINVAL; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (!st->id) @@ -1343,7 +1440,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { - size_t n = src->allocated_stack / BPF_REG_SIZE; + size_t n = bpf_stack_nr_slots(src); dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), GFP_KERNEL_ACCOUNT); @@ -1380,9 +1477,9 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n) /* Possibly update state->allocated_stack to be at least size bytes. Also * possibly update the function's high-water mark in its bpf_subprog_info. */ -static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) { - size_t old_n = state->allocated_stack / BPF_REG_SIZE, n; + size_t old_n = bpf_stack_nr_slots(state), n; /* The stack size is always a multiple of BPF_REG_SIZE. */ size = round_up(size, BPF_REG_SIZE); @@ -1603,6 +1700,7 @@ static void free_func_state(struct bpf_func_state *state) { if (!state) return; + kfree(state->loop_stack); kfree(state->stack_arg_regs); kfree(state->stack); kfree(state); @@ -1639,6 +1737,10 @@ static int copy_func_state(struct bpf_func_state *dst, memcpy(dst, src, offsetof(struct bpf_func_state, stack)); /* Instruction accounting is path-local, not part of verifier state. */ dst->insns_subtotal = 0; + dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt, + sizeof(*src->loop_stack), GFP_KERNEL_ACCOUNT); + if (!dst->loop_stack) + return -ENOMEM; return copy_stack_state(dst, src); } @@ -1667,6 +1769,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, return err; dst_state->speculative = src->speculative; dst_state->in_sleepable = src->in_sleepable; + dst_state->unwound = src->unwound; dst_state->curframe = src->curframe; dst_state->branches = src->branches; dst_state->parent = src->parent; @@ -1846,12 +1949,124 @@ static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, } } +static void reg_step_reset(struct bpf_reg_state *reg) +{ + reg->base = 0; + reg->step = 1; +} + +static void scalar_step_add(struct bpf_reg_state *dst_reg, + const struct bpf_reg_state *a, + const struct bpf_reg_state *b) +{ + const struct bpf_reg_state *reg; + s64 amount, tmp; + + if (tnum_is_const(b->var_off)) { + reg = a; + amount = (s64)b->var_off.value; + } else if (tnum_is_const(a->var_off)) { + reg = b; + amount = (s64)a->var_off.value; + } else { + reg_step_reset(dst_reg); + return; + } + + /* + * Let x be a possible signed value of the register, + * d = amount, s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * If the addition causes signed underflow, the resulting value + * is x + d + 2**64. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s (1) + * Keeping reg->step = s and updating reg->base to (b + d) mod s + * requires (1) to equal (b + d) mod s, which holds if and only + * if 2**64 mod s = 0. + * Signed overflow subtracts 2**64 and has the same requirement. + * + * Conservatively reset the stride if either wrap is possible. + */ + if (check_add_overflow(reg_smin(reg), amount, &tmp) || + check_add_overflow(reg_smax(reg), amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + + dst_reg->base = (reg->base + imod(amount, reg->step)) % reg->step; + dst_reg->step = reg->step; +} + +static void scalar_step_scale(struct bpf_reg_state *dst_reg, u64 amount) +{ + u16 step; + s64 tmp; + + /* + * Using the same notation as scalar_step_add(), with d > 0, + * the scaled value satisfies (x * d) mod (s * d) = b * d. + * If the multiplication wraps, its signed result can be represented + * as x * d - n * 2**64 for some nonzero integer n. + * + * Taking the result modulo s * d gives: + * (x * d - n * 2**64) mod (s * d) = + * (b * d - (n * 2**64 mod (s * d))) mod (s * d) (1) + * Updating dst_reg->base to b * d and dst_reg->step to s * d + * requires (1) to equal b * d, which holds if and only if + * n * 2**64 mod (s * d) = 0. + * + * Conservatively reset the stride if a wrap is possible. + */ + if (amount == 0 || check_mul_overflow(dst_reg->step, amount, &step) || + check_mul_overflow(reg_smin(dst_reg), (s64)amount, &tmp) || + check_mul_overflow(reg_smax(dst_reg), (s64)amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + /* + * The above established that `step = dst_reg->step * amount` + * does not overflow, therefore the base < step invariant holds + * after assignments below. + */ + dst_reg->base = dst_reg->base * amount; + dst_reg->step = step; +} + +/* + * Let x be a possible signed value of the register, + * s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * Truncating to N bits and then zero- or sign-extending x + * is equivalent to computing x + k * 2**N for some integer k. + * For example, sign-extending the low 8 bits of x = 255 can + * be expressed as 255 - 2**8 = -1. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + k * 2**N) mod s = (b + (k * 2**N mod s)) mod s + * Keeping reg->base = b and reg->step = s requires this to equal b, + * which holds if and only if k * 2**N mod s = 0. + * Continuing the example with s = 3 and b = 0, we have 255 mod 3 = 0, + * but (-1) mod 3 = 2, so the original base and step no longer hold. + * + * Conservatively reset the stride unless all possible values fit in + * the range where the conversion leaves them unchanged. + * Call before updating the register's bounds. Requires 0 < bits < 64. + */ +static void reg_step_check_unsigned(struct bpf_reg_state *reg, u32 bits) +{ + if (reg_umax(reg) >= (1ULL << bits)) + reg_step_reset(reg); +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const(imm); reg->r64 = cnum64_from_urange(imm, imm); reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); + reg_step_reset(reg); } /* Mark the unknown part of a register (variable offset or scalar value) as @@ -1868,6 +2083,12 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) ___mark_reg_known(reg, imm); } +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm) +{ + __mark_reg_known(reg, imm); + reg->type = SCALAR_VALUE; +} + static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const_subreg(reg->var_off, imm); @@ -2109,8 +2330,24 @@ static void __reg_bound_offset(struct bpf_reg_state *reg) struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), tnum_range(reg_u32_min(reg), reg_u32_max(reg))); + u32 trailing_zero_bits; + u16 base = reg->base; + u16 step = reg->step; reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); + + if (base == 0) + trailing_zero_bits = count_trailing_zeros(step); + else + trailing_zero_bits = min(count_trailing_zeros(base), + count_trailing_zeros(step)); + reg->var_off = tnum_and(reg->var_off, tnum_const(~0ULL << trailing_zero_bits)); +} + +static void __update_bounds_from_step(struct bpf_reg_state *reg) +{ + reg->r64 = cnum64_intersect_linear(reg->r64, reg->base, reg->step); + deduce_bounds_32_from_64(reg); } static bool range_bounds_violation(struct bpf_reg_state *reg); @@ -2132,6 +2369,15 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) * then new var_off is (0; 0x7f...fc) which improves our umax. */ __update_reg_bounds(reg); + /* + * Both tnum and base+step define a sequence of points, + * potentially w/o intersection, or intersecting after N + * tnum_step()'s, where N can be large. On the other hand, + * step+base *must* be in sync with r64 in order for + * array traversal logic starting at reg_smin() to be sound. + * Hence, place this as a final step in the reg_bounds_sync(). + */ + __update_bounds_from_step(reg); } static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) @@ -2175,17 +2421,24 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, goto out; } + if (reg->step == 0 || reg->base >= reg->step) { + msg = "invalid base/step"; + goto out; + } + return 0; out: verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " - "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)", + "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u", ctx, msg, reg->r64.base, reg->r64.size, reg->r32.base, reg->r32.size, - reg->var_off.value, reg->var_off.mask); + reg->var_off.value, reg->var_off.mask, + reg->base, reg->step); if (env->test_reg_invariants) return -EFAULT; __mark_reg_unbounded(reg); + reg_step_reset(reg); return 0; } @@ -2196,6 +2449,7 @@ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; __mark_reg_unbounded(reg); + reg_step_reset(reg); } /* Mark a register as having a completely unknown (scalar) value, @@ -2406,6 +2660,7 @@ static int add_subprog(struct bpf_verifier_env *env, int off) return -E2BIG; } /* determine subprog starts. The end is one before the next starts */ + env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; env->subprog_info[env->subprog_cnt++].start = off; sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL); @@ -2514,17 +2769,21 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - return bsearch(&desc, tab->descs, tab->nr_descs, + return bsearch(&desc, tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); } int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { + struct bpf_kfunc_desc_tab *tab; const struct bpf_kfunc_desc *desc; - desc = find_kfunc_desc(prog, func_id, btf_fd_idx); - if (!desc) + tab = prog->aux->kfunc_tab; + if (desc_idx >= tab->nr_descs) + return -EFAULT; + desc = &tab->descs[desc_idx]; + if (desc->func_id != func_id) return -EFAULT; *func_addr = (u8 *)desc->addr; @@ -2799,7 +3058,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto); + const struct btf_func_model *fm, struct bpf_func_proto *proto); int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { @@ -2864,10 +3123,15 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (find_kfunc_desc(env->prog, func_id, offset)) return 0; - if (tab->nr_descs == MAX_KFUNC_DESCS) { + if (tab->nr_base_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; } + if (tab->nr_descs != tab->nr_base_descs) { + verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)", + tab->nr_descs, tab->nr_base_descs); + return -EFAULT; + } err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) @@ -2888,6 +3152,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; @@ -2904,7 +3180,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc = &tab->descs[tab->nr_descs]; memset(desc, 0, sizeof(*desc)); - err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto); if (err) return err; @@ -2913,7 +3189,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc->addr = addr; desc->func_model = func_model; tab->nr_descs++; - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), + tab->nr_base_descs++; + sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } @@ -2955,6 +3232,22 @@ static int add_subprogs(struct bpf_verifier_env *env) return ret; } + /* + * func_info describes all functions of the program. Those that are + * referenced only from data, e.g. from a table of functions to be + * called via callx, are not seen by the loop above. They are possible + * callees that have to be known upfront as well. + */ + if (env->bpf_capable) { + struct bpf_prog_aux *aux = env->prog->aux; + + for (i = 1; i < aux->func_info_cnt; i++) { + ret = add_subprog(env, aux->func_info[i].insn_off); + if (ret < 0) + return ret; + } + } + ret = bpf_find_exception_callback_insn_off(env); if (ret < 0) return ret; @@ -3014,6 +3307,34 @@ static int add_kfuncs(struct bpf_verifier_env *env) return 0; } +static void find_subprog_properties(struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog = env->subprog_info; + struct bpf_insn *insn = env->prog->insnsi; + int cur_subprog; + + for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) { + int i; + + for (i = subprog[cur_subprog].start; + i < subprog[cur_subprog + 1].start; i++) { + u8 code = insn[i].code; + + if (code == (BPF_JMP | BPF_CALL) && + insn[i].src_reg == 0 && + insn[i].imm == BPF_FUNC_tail_call) { + subprog[cur_subprog].has_tail_call = true; + subprog[cur_subprog].tail_call_reachable = true; + } + if (BPF_CLASS(code) == BPF_LD && + (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) + subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; + } + } +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -3027,15 +3348,6 @@ static int check_subprogs(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++) { u8 code = insn[i].code; - if (code == (BPF_JMP | BPF_CALL) && - insn[i].src_reg == 0 && - insn[i].imm == BPF_FUNC_tail_call) { - subprog[cur_subprog].has_tail_call = true; - subprog[cur_subprog].tail_call_reachable = true; - } - if (BPF_CLASS(code) == BPF_LD && - (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) - subprog[cur_subprog].has_ld_abs = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -3059,9 +3371,10 @@ static int check_subprogs(struct bpf_verifier_env *env) } next: if (i == subprog_end - 1) { - /* to avoid fall-through from one subprog into another + /* + * To avoid fall-through from one subprog into another, * the last insn of the subprog should be either exit - * or unconditional jump back or bpf_throw call + * or unconditional jump back or bpf_throw call. */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && @@ -3085,11 +3398,89 @@ static int check_subprogs(struct bpf_verifier_env *env) } /* + * The callee of callx is known to the main verification pass only, which + * records the 'caller' -> 'callee' edge of the call graph for the checks + * that follow it: absence of recursion and the maximum stack depth. + */ +static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee) +{ + u32 cnt = env->subprog_cnt; + + if (!env->callx_edges) { + env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long), + GFP_KERNEL_ACCOUNT); + if (!env->callx_edges) + return -ENOMEM; + } + __set_bit(caller * cnt + callee, env->callx_edges); + return 0; +} + +/* + * Return the first subprog with the number >= 'from' that 'caller' calls + * via callx, or -1 when there is none. + */ +static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from) +{ + u32 cnt = env->subprog_cnt; + unsigned long bit, end = (caller + 1) * cnt; + + if (!env->callx_edges || from >= cnt) + return -1; + bit = find_next_bit(env->callx_edges, end, caller * cnt + from); + return bit < end ? bit - caller * cnt : -1; +} + +/* + * 'call foo; unwind_goto pad' goes to the pad when foo left with unwind_exit. + * JITs don't emit a branch for it. unwind_exit finds the pad by its return + * address. So unwind_goto has to follow the call and nothing may jump to it. + */ +static int check_unwind_insns(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + bool seen = false; + int i; + + for (i = 0; i < env->prog->len; i++, insn++) { + if (bpf_is_unwind_exit(insn)) { + bpf_find_containing_subprog(env, i)->can_unwind = true; + seen = true; + } + if (!bpf_is_unwind_goto(insn)) + continue; + seen = true; + if (!i || !(bpf_pseudo_call(insn - 1) || bpf_is_callx(insn - 1))) { + verbose(env, "unwind_goto at insn %d doesn't follow a call of a subprog\n", + i); + return -EINVAL; + } + if (bpf_is_jump_target(env, i)) { + verbose(env, "jump to unwind_goto at insn %d\n", i); + return -EINVAL; + } + } + if (!seen) + return 0; + if (!bpf_jit_supports_unwind() || bpf_prog_is_offloaded(env->prog->aux)) { + verbose(env, "JIT does not support unwinding\n"); + return -ENOTSUPP; + } + env->prog->jit_required = 1; + env->prog->aux->can_unwind = env->subprog_info[0].can_unwind; + return 0; +} + +/* * Sort subprogs in topological order so that leaf subprogs come first and * their callers come later. This is a DFS post-order traversal of the call * graph. Scan only reachable instructions (those in the computed postorder) of * the current subprog to discover callees (direct subprogs and sync * callbacks). + * + * The callees of callx are not known before the main verification pass. + * When callx is used the sort is repeated after it with the recorded callx + * edges added to the call graph to reject recursion through indirect calls. */ static int sort_subprogs_topo(struct bpf_verifier_env *env) { @@ -3130,12 +3521,22 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) int idx = insn_postorder[j]; int callee; - if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx])) + if (bpf_is_callx(&insn[idx])) { + /* find a callee that is not explored yet */ + callee = -1; + do { + callee = next_callx_callee(env, cur, callee + 1); + } while (callee >= 0 && color[callee] == 2); + if (callee < 0) + continue; + } else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) { + callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); + if (callee < 0) { + ret = -EFAULT; + goto out; + } + } else { continue; - callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); - if (callee < 0) { - ret = -EFAULT; - goto out; } if (color[callee] == 2) continue; @@ -3232,26 +3633,30 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + +/* Layout of one packed linked register in the jump history, see linked_regs_pack() */ #define LR_FRAMENO_BITS 4 -#define LR_SPI_BITS 6 -#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) -#define LR_SIZE_BITS 4 -#define LR_FRAMENO_MASK ((1ull << LR_FRAMENO_BITS) - 1) -#define LR_SPI_MASK ((1ull << LR_SPI_BITS) - 1) -#define LR_SIZE_MASK ((1ull << LR_SIZE_BITS) - 1) -#define LR_SPI_OFF LR_FRAMENO_BITS -#define LR_IS_REG_OFF (LR_SPI_BITS + LR_FRAMENO_BITS) -#define LINKED_REGS_MAX 5 +#define LR_INDEX_BITS 11 +#define LR_FRAMENO_MASK ((1u << LR_FRAMENO_BITS) - 1) +#define LR_IS_REG BIT(LR_FRAMENO_BITS) +#define LR_INDEX_OFF (LR_FRAMENO_BITS + 1) +#define LR_INDEX_MASK ((1u << LR_INDEX_BITS) - 1) +#define LINKED_REGS_MAX BPF_LINKED_REGS_MAX static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS)); -static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS)); -static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64); +static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS)); +static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16); struct linked_reg { u8 frameno; union { - u8 spi; - u8 regno; + u16 spi; + u16 regno; }; bool is_reg; }; @@ -3270,48 +3675,34 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s) } /* - * Use u64 as a vector of 5 11-bit values, use first 4-bits to track - * number of elements currently in stack. - * Pack one history entry for linked registers as 11 bits in the following format: - * - 4-bits frameno - * - 6-bits spi_or_reg - * - 1-bit is_reg + * Pack linked registers for a jump history entry, one u16 each: + * - 4 bits frameno + * - 1 bit is_reg + * - 11 bits register or stack slot index */ -static u64 linked_regs_pack(struct linked_regs *s) +static void linked_regs_pack(const struct linked_regs *s, u16 *packed) { - u64 val = 0; int i; for (i = 0; i < s->cnt; ++i) { - struct linked_reg *e = &s->entries[i]; - u64 tmp = 0; + const struct linked_reg *e = &s->entries[i]; - tmp |= e->frameno; - tmp |= e->spi << LR_SPI_OFF; - tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF; - - val <<= LR_ENTRY_BITS; - val |= tmp; + packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF); } - val <<= LR_SIZE_BITS; - val |= s->cnt; - return val; } -static void linked_regs_unpack(u64 val, struct linked_regs *s) +static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s) { int i; - s->cnt = val & LR_SIZE_MASK; - val >>= LR_SIZE_BITS; - + s->cnt = hist->linked_regs_cnt; for (i = 0; i < s->cnt; ++i) { struct linked_reg *e = &s->entries[i]; + u16 packed = hist->linked_regs[i]; - e->frameno = val & LR_FRAMENO_MASK; - e->spi = (val >> LR_SPI_OFF) & LR_SPI_MASK; - e->is_reg = (val >> LR_IS_REG_OFF) & 0x1; - val >>= LR_ENTRY_BITS; + e->frameno = packed & LR_FRAMENO_MASK; + e->is_reg = packed & LR_IS_REG; + e->spi = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK; } } @@ -3353,10 +3744,10 @@ void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_ bool some_precise = false; int i; - if (!hist || hist->linked_regs == 0) + if (!hist || !hist->linked_regs_cnt) return; - linked_regs_unpack(hist->linked_regs, &linked_regs); + linked_regs_unpack(hist, &linked_regs); for (i = 0; i < linked_regs.cnt; ++i) { struct linked_reg *e = &linked_regs.entries[i]; @@ -3453,17 +3844,18 @@ static void save_register_state(struct bpf_verifier_env *env, int spi, struct bpf_reg_state *reg, int size) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; - bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); - state->stack[spi].spilled_ptr = *reg; + bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL); + slot->spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) - state->stack[spi].slot_type[i - 1] = STACK_SPILL; + slot->slot_type[i - 1] = STACK_SPILL; /* size < 8 bytes spill */ for (; i; i--) - mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + mark_stack_slot_misc(env, &slot->slot_type[i - 1]); bpf_diag_mod_end(env); } @@ -3505,14 +3897,15 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env, static void scrub_special_slot(struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; /* regular write of data into stack destroys any spilled ptr */ - state->stack[spi].spilled_ptr.type = NOT_INIT; + slot->spilled_ptr.type = NOT_INIT; /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ - if (is_stack_slot_special(&state->stack[spi])) + if (is_stack_slot_special(slot)) for (i = 0; i < BPF_REG_SIZE; i++) - scrub_spilled_slot(&state->stack[spi].slot_type[i]); + scrub_spilled_slot(&slot->slot_type[i]); } /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, @@ -3530,13 +3923,15 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg = NULL; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(state, spi); - /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, + /* + * caller checked that off % size == 0 and -env->stack_limit <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && - bpf_is_spilled_reg(&state->stack[spi]) && - !bpf_is_spilled_scalar_reg(&state->stack[spi]) && + bpf_is_spilled_reg(ss) && + !bpf_is_spilled_scalar_reg(ss) && size != BPF_REG_SIZE) { const char *reason; @@ -3558,7 +3953,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[(slot - i) % + u8 type = ss->slot_type[(slot - i) % BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { @@ -3575,7 +3970,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (err) return err; - check_fastcall_stack_contract(env, state, insn_idx, off); mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { bool reg_value_fits; @@ -3587,7 +3981,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) - clear_scalar_id(&state->stack[spi].spilled_ptr); + clear_scalar_id(&ss->spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; @@ -3611,8 +4005,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; - if (bpf_is_spilled_reg(&state->stack[spi])) - bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + if (bpf_is_spilled_reg(ss)) + bpf_diag_record_scrub(env, &ss->spilled_ptr, BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); @@ -3633,13 +4027,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) - state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; + ss->slot_type[(slot - i) % BPF_REG_SIZE] = type; insn_flags = 0; /* not a register spill */ } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -3687,24 +4081,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) writing_zero = true; - for (i = min_off; i < max_off; i++) { - int spi; + err = destroy_dynptrs_in_stack_slots(env, state, bpf_get_spi(max_off - 1), + bpf_get_spi(min_off)); + if (err) + return err; - spi = bpf_get_spi(i); - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - } - - check_fastcall_stack_contract(env, state, insn_idx, min_off); /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { + struct bpf_stack_state *ss; u8 new_type, *stype; int slot, spi; slot = -i - 1; spi = slot / BPF_REG_SIZE; - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; mark_stack_slot_scratched(env, spi); if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { @@ -3727,9 +4118,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, /* If writing_zero and the spi slot contains a spill of value 0, * maintain the spill type. */ - if (writing_zero && *stype == STACK_SPILL && - bpf_is_spilled_scalar_reg(&state->stack[spi])) { - struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; + if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) { + struct bpf_reg_state *spill_reg = &ss->spilled_ptr; if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { zero_used = true; @@ -3800,23 +4190,22 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; - u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; - int zeros = 0; + int zeros = 0, zero_spills = 0; for (i = min_off; i < max_off; i++) { slot = -i - 1; spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); - stype = ptr_state->stack[spi].slot_type; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { zeros++; continue; } if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && - bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { - zero_spill_mask |= 1ull << spi; + bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) { + zero_spills++; zeros++; continue; } @@ -3827,8 +4216,14 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); - if (zero_spill_mask) { - bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + if (zero_spills) { + for (i = min_off; i < max_off; i++) { + slot = -i - 1; + spi = slot / BPF_REG_SIZE; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL) + bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi); + } return mark_chain_precision_batch(env, env->cur_state); } } else { @@ -3839,19 +4234,46 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, return 0; } -static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, - int size) +static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size, + enum bpf_stack_slot_type type) { - const char *reason; + const char *problem, *reason, *suggestion, *kind; - reason = bpf_diag_fmt(env, - "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " - "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", - size, off, i); - bpf_diag_memory( - env, env->insn_idx, "uninitialized stack read", reason, - "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " - "or load with CAP_PERFMON if uninitialized stack reads are intended."); + if (type == STACK_INVALID) { + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); + return; + } + + switch (type) { + case STACK_DYNPTR: + kind = "dynptr"; + suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly."; + break; + case STACK_ITER: + kind = "iterator"; + suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly."; + break; + case STACK_IRQ_FLAG: + kind = "IRQ flag"; + suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly."; + break; + default: + return; + } + + problem = bpf_diag_fmt(env, "direct read of %s stack state", kind); + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. " + "This state has an opaque representation that BPF programs cannot read directly.", + size, off, i, kind); + bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion); } /* Read the stack at 'off' and put the results into the register indicated by @@ -3876,12 +4298,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi); - stype = reg_state->stack[spi].slot_type; - reg = ®_state->stack[spi].spilled_ptr; + stype = ss->slot_type; + reg = &ss->spilled_ptr; mark_stack_slot_scratched(env, spi); - check_fastcall_stack_contract(env, state, env->insn_idx, off); /* * Refine the in-progress load record's origin to the source stack slot. @@ -3889,11 +4311,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (dst_regno >= 0) bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); - if (bpf_is_spilled_reg(®_state->stack[spi])) { - u8 spill_size = 1; - - for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) - spill_size++; + if (bpf_is_spilled_reg(ss)) { + u8 spill_size = bpf_spill_size(ss); if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { if (reg->type != SCALAR_VALUE) { @@ -3902,11 +4321,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, return -EACCES; } - if (dst_regno < 0) - return 0; - if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { + if (dst_regno < 0) + return 0; + if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -3943,11 +4362,14 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } + if (dst_regno < 0) + return 0; + if (spill_cnt == size && tnum_is_const(reg->var_off) && reg->var_off.value == 0) { __mark_reg_const_zero(env, &state->regs[dst_regno]); @@ -4002,7 +4424,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } @@ -4015,7 +4437,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -4068,7 +4490,6 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg dst_regno); if (err) return err; - check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -4101,13 +4522,13 @@ static int check_stack_read(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); - reason = bpf_diag_fmt(env, - "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " - "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + reason = bpf_diag_fmt( + env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. " + "This stack access requires a constant stack offset and a precise initialized range.", tn_buf, off, size); bpf_diag_memory( - env, env->insn_idx, "variable stack access", reason, - "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); + env, env->insn_idx, "variable-offset stack access", reason, + "Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -4205,7 +4626,7 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } /* @@ -4239,7 +4660,7 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st cur->regs[dst_regno] = *arg; bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) @@ -4599,7 +5020,7 @@ BTF_ID(struct, cgroup) BTF_ID(struct, bpf_cpumask) #endif BTF_ID(struct, task_struct) -#ifdef CONFIG_CRYPTO +#ifdef CONFIG_BPF_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif #ifdef CONFIG_INET @@ -4822,7 +5243,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_func_proto *fn, + const struct bpf_call_arg_meta *meta, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4846,10 +5267,11 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (fn) - return fn->pkt_access; + if (meta && !meta->btf && meta->func_id) + return meta->fn->pkt_access; - env->seen_direct_write = true; + if (t == BPF_WRITE) + env->seen_direct_write = true; return true; case BPF_PROG_TYPE_CGROUP_SOCKOPT: @@ -5061,6 +5483,19 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +/* + * There are no address spaces in Rust, addresses of arena are plain numbers. + * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or + * a store through a number is an access to arena at the low 32 bits of it, + * like the access through PTR_TO_ARENA is. The register stays a number. + */ +static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; +} + static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5091,7 +5526,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno)) + if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -5107,27 +5542,17 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; -static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) -{ - enum bpf_reg_type type; - - for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { - if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) - return type; - } - - return NOT_INIT; -} - -static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) +static bool is_trusted_reg_type(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + enum bpf_reg_type type) { /* A referenced register is always trusted. */ if (reg_is_referenced(env, reg)) return true; /* Types listed in the reg2btf_ids are always trusted */ - if (reg2btf_ids[base_type(reg->type)] && - !bpf_type_has_unsafe_modifiers(reg->type)) + if (reg2btf_ids[base_type(type)] && + !bpf_type_has_unsafe_modifiers(type)) return true; /* If a register is not referenced, it is trusted if it has the @@ -5139,8 +5564,14 @@ static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_st * Eventually, we should make PTR_TRUSTED the single source of truth * for whether a register is trusted. */ - return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && - !bpf_type_has_unsafe_modifiers(reg->type); + return type_flag(type) & BPF_REG_TRUSTED_MODIFIERS && + !bpf_type_has_unsafe_modifiers(type); +} + +static bool is_trusted_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return is_trusted_reg_type(env, reg, reg->type); } static bool is_rcu_reg(const struct bpf_reg_state *reg) @@ -5313,6 +5744,9 @@ struct bpf_subprog_call_depth_info { int ret_insn; /* caller instruction where we return to. */ int caller; /* caller subprogram idx */ int frame; /* # of consecutive static call stack frames on top of stack */ + int callx_insn; /* callx instruction whose callees are being walked */ + int callx_next; /* next callee of callx_insn to walk */ + bool via_callx; /* the subprogram is entered via callx */ }; /* starting from main bpf function walk all instructions of the function @@ -5332,11 +5766,13 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, /* no caller idx */ dinfo[idx].caller = -1; + dinfo[idx].via_callx = false; i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + dinfo[idx].callx_insn = -1; if (subprog[idx].has_ld_abs) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { @@ -5389,7 +5825,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { if (subprog_depth > env->max_stack_depth) env->max_stack_depth = subprog_depth; - if (subprog_depth > MAX_BPF_STACK) { + if (subprog_depth > env->stack_limit) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); return -EACCES; @@ -5398,7 +5834,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, depth += subprog_depth; if (depth > env->max_stack_depth) env->max_stack_depth = depth; - if (depth > MAX_BPF_STACK) { + if (depth > env->stack_limit) { total = 0; for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) total++; @@ -5413,11 +5849,10 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, for (; i < subprog_end; i++) { int next_insn, sidx; - if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { + if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off && + bpf_is_throw_kfunc(insn + i)) { bool err = false; - if (!bpf_is_throw_kfunc(insn + i)) - continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { err = true; @@ -5432,31 +5867,70 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, return -EINVAL; } - if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) - continue; - /* remember insn and function to return to */ - - /* find the callee */ - next_insn = i + insn[i].imm + 1; - sidx = bpf_find_subprog(env, next_insn); - if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) - return -EFAULT; - if (subprog[sidx].is_async_cb) { - /* async callbacks don't increase bpf prog stack size unless called directly */ - if (!bpf_pseudo_call(insn + i)) - continue; - if (subprog[sidx].is_exception_cb) { - verbose(env, "insn %d cannot call exception cb directly", i); - return -EINVAL; + if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) { + /* + * Walk the callees and the callbacks recorded by + * the main verification pass one by one, returning to + * this insn after each. + */ + if (dinfo[idx].callx_insn != i) { + dinfo[idx].callx_insn = i; + dinfo[idx].callx_next = 0; } + sidx = next_callx_callee(env, idx, dinfo[idx].callx_next); + if (sidx < 0) + continue; + dinfo[idx].callx_next = sidx + 1; + dinfo[idx].ret_insn = i; + next_insn = subprog[sidx].start; + } else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) { + /* find the callee */ + next_insn = i + insn[i].imm + 1; + sidx = bpf_find_subprog(env, next_insn); + if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) + return -EFAULT; + if (subprog[sidx].is_async_cb) { + /* async callbacks don't increase bpf prog stack size unless called directly */ + if (!bpf_pseudo_call(insn + i)) + continue; + if (subprog[sidx].is_exception_cb) { + verbose(env, "insn %d cannot call exception cb directly", i); + return -EINVAL; + } + } + /* remember insn to return to */ + dinfo[idx].ret_insn = i + 1; + } else { + continue; + } + + /* + * sort_subprogs_topo() tolerates cycles in the call graph that + * go through the address of a function being taken, since it + * doesn't know what it is taken for. Such cycle is a recursion + * unless it's an async callback, which are skipped above. + * The main verification pass limits the depth of the recursion, + * but it doesn't follow calls of global functions. + */ + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (tmp != sidx) + continue; + verbose(env, "recursive call from %s() to %s()\n", + bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx)); + bpf_diag_program_structure( + env, i, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This call or callback would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); + return -EINVAL; } /* store caller info for after we return from callee */ dinfo[idx].frame = frame; - dinfo[idx].ret_insn = i + 1; /* push caller idx into callee's dinfo */ dinfo[sidx].caller = idx; + dinfo[sidx].via_callx = bpf_is_callx(insn + i); i = next_insn; @@ -5490,6 +5964,14 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; } + /* + * JITs pass tail call counter in a register that is + * not available when the callee is called via callx. + */ + if (dinfo[tmp].via_callx) { + verbose(env, "tail_calls are not allowed in functions called via callx\n"); + return -EINVAL; + } subprog[tmp].tail_call_reachable = true; } } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { @@ -5625,9 +6107,10 @@ static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, - u32 *max_access) + u32 *max_access, bool write) { const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; + struct bpf_prog_aux *aux = env->prog->aux; u32 access_end; int err; @@ -5636,6 +6119,11 @@ static int check_buffer_access(struct bpf_verifier_env *env, return err; *max_access = max(access_end, *max_access); + if (write && size) { + if (!aux->max_rdwr_write || access_end - size < aux->min_rdwr_write) + aux->min_rdwr_write = access_end - size; + aux->max_rdwr_write = max(access_end, aux->max_rdwr_write); + } return 0; } @@ -5643,6 +6131,7 @@ static int check_buffer_access(struct bpf_verifier_env *env, /* BPF architecture zero extends alu32 ops into 64-bit registesr */ static void zext_32_to_64(struct bpf_reg_state *reg) { + reg_step_check_unsigned(reg, 32); reg->var_off = tnum_subreg(reg->var_off); reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); } @@ -5652,13 +6141,14 @@ static void zext_32_to_64(struct bpf_reg_state *reg) */ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) { - u64 mask; + u64 mask = (1ULL << (size * 8)) - 1; + + reg_step_check_unsigned(reg, size * 8); /* clear high bits in bit representation */ reg->var_off = tnum_cast(reg->var_off, size); /* fix arithmetic bounds */ - mask = ((u64)1 << (size * 8)) - 1; if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); else @@ -5695,6 +6185,14 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; u64 top_smax_value, top_smin_value; u64 num_bits = size * 8; + s64 limit = 1LL << (num_bits - 1); + + /* If every value fits the signed N-bit range sign extension is a noop. */ + if (reg_smin(reg) >= -limit && reg_smax(reg) < limit) + return; + + /* See commen at reg_step_check_unsigned(). */ + reg_step_reset(reg); if (tnum_is_const(reg->var_off)) { u64_cval = reg->var_off.value; @@ -5761,6 +6259,16 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) u32 top_smax_value, top_smin_value; u32 num_bits = size * 8; + /* + * If value fits the non-negative half of the signed N-bit range, + * sign extension is a noop. + */ + if (reg_umax(reg) < (1ULL << (num_bits - 1))) + return; + + /* See comment at reg_step_check_unsigned(). */ + reg_step_reset(reg); + if (tnum_is_const(reg->var_off)) { u32_val = reg->var_off.value; if (size == 1) @@ -5854,6 +6362,110 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, return 0; } +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct bpf_func_ptr *a = _a, *b = _b; + + if (a->map != b->map) + return a->map < b->map ? -1 : 1; + if (a->map_off != b->map_off) + return a->map_off < b->map_off ? -1 : 1; + return 0; +} + +/* Find the first pointer to a function at or after 'off' in the value of 'map' */ +static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off) +{ + u32 l = 0, r = env->func_ptr_cnt, m; + struct bpf_func_ptr *p; + + while (l < r) { + m = l + (r - l) / 2; + p = &env->func_ptrs[m]; + if (p->map < map || (p->map == map && p->map_off < off)) + l = m + 1; + else + r = m; + } + return l; +} + +/* + * Return pointers to functions that overlap with 'size' bytes at offset 'off' + * of the value of 'map' and their number in 'cnt'. + */ +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt) +{ + u32 first, last; + + *cnt = 0; + if (!env->func_ptr_cnt || !size) + return NULL; + + /* a pointer that starts up to 7 bytes before 'off' overlaps too */ + first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0); + last = func_ptr_lower_bound(env, map, off + size); + if (first >= last) + return NULL; + + *cnt = last - first; + return &env->func_ptrs[first]; +} + +/* Return all pointers to functions in the value of 'map' */ +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt) +{ + return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt); +} + +/* instructions [off, off + len) replaced the instruction at 'off' */ +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + u32 i; + + if (len <= 1) + return; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + p->xlated_off += len - 1; + } +} + +/* + * Instructions [off, off + len) are about to be removed. It's called before + * the starts of subprogs are adjusted. A subprog is gone when all of its + * instructions are. Otherwise, e.g. when its first instruction is a nop, + * it starts where the removed instructions did. + */ +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + int subprog; + u32 i; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + if (p->xlated_off >= off + len) { + p->xlated_off -= len; + continue; + } + subprog = bpf_find_subprog(env, p->xlated_off); + if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len) + p->xlated_off = off; + else + p->xlated_off = BPF_FUNC_PTR_DELETED; + } +} + #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) @@ -5923,10 +6535,36 @@ BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode; }; +/* + * The pointer fields in the sched_ext ops argument containers are pinned by the + * callers for the duration of the ops calls and are never NULL. + */ +BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { +#ifdef CONFIG_EXT_GROUP_SCHED + struct cgroup *cgroup; +#endif +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { + struct task_struct *task; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { + struct sched_ext_ops *ops; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { + struct sched_ext_ops *ops; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; +BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) { + struct mm_struct *mm; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { struct sock *sk; }; @@ -5967,6 +6605,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env, BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); } @@ -5977,12 +6619,52 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, { BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted_or_null"); } +/* + * The member holds the kernel address of arena memory, which the rewritten + * load turns into an arena pointer, see arena_member_fixup(). NULL needs no + * tracking, PTR_TO_ARENA covers it. + */ +static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int value_regno, enum bpf_type_flag flag, const char *tname, + const char *field_name) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; + struct bpf_insn *insn; + + /* a helper reading the member as bytes needs no rewrite */ + if (value_regno < 0) + return 0; + if (!env->prog->aux->arena) { + verbose(env, + "arena pointer member %s of struct %s requires a program with an associated arena\n", + field_name, tname); + return -EINVAL; + } + /* only a plain load or an acquire load is rewritten */ + insn = &env->prog->insnsi[env->insn_idx]; + if (BPF_CLASS(insn->code) != BPF_LDX && + !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { + verbose(env, + "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", + field_name, tname); + return -EACCES; + } + + aux->arena_member = true; + if (flag & PTR_MAYBE_NULL) + aux->arena_member_nullable = true; + mark_reg_unknown(env, regs, value_regno); + regs[value_regno].type = PTR_TO_ARENA; + return 0; +} + static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, @@ -5994,6 +6676,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, const char *field_name = NULL; enum bpf_type_flag flag = 0; u32 btf_id = 0; + s64 min_off; int ret; if (!env->allow_ptr_leaks) { @@ -6009,36 +6692,31 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EINVAL; } - if (!tnum_is_const(reg->var_off)) { - char tn_buf[48]; - - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + if (check_add_overflow(reg_smin(reg), off, &min_off)) { verbose(env, - "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", - reg_arg_name(env, argno), tname, off, tn_buf); + "%s is ptr_%s access, offset computation overflows: register's minimal offset is %lld, instruction offset is %d\n", + reg_arg_name(env, argno), tname, reg_smin(reg), off); return -EACCES; } - off += reg->var_off.value; - - if (off < 0) { + if (min_off < 0) { verbose(env, - "%s is ptr_%s invalid negative access: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s invalid negative access: off=%lld\n", + reg_arg_name(env, argno), tname, min_off); return -EACCES; } if (reg->type & MEM_USER) { verbose(env, - "%s is ptr_%s access user memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access user memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } if (reg->type & MEM_PERCPU) { verbose(env, - "%s is ptr_%s access percpu memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access percpu memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } @@ -6048,6 +6726,18 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, } if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { + if (!tnum_is_const(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, + "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", + reg_arg_name(env, argno), tname, off, tn_buf); + return -EACCES; + } + + off += reg->var_off.value; + if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); return -EFAULT; @@ -6084,6 +6774,9 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret < 0) return ret; + if (ret == PTR_TO_ARENA) + return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); + if (ret != PTR_TO_BTF_ID) { /* just mark; */ @@ -6144,6 +6837,16 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, clear_trusted_flags(&flag); } + /* + * A __nullable member only matters where NULL would otherwise be + * assumed away, which is anywhere but on a plain or an untrusted + * pointer: those are probed on a dereference, which tolerates NULL, and + * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A + * per-CPU pointer counts, per_cpu_ptr() does not check for NULL. + */ + if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL))) + flag &= ~PTR_MAYBE_NULL; + if (atype == BPF_READ && value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) @@ -6218,10 +6921,11 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, return 0; } -/* Check that the stack access at the given offset is within bounds. The +/* + * Check that the stack access at the given offset is within bounds. The * maximum valid offset is -1. * - * The minimum valid offset is -MAX_BPF_STACK for writes, and + * The minimum valid offset is -env->stack_limit for writes, and * -state->allocated_stack for reads. */ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, @@ -6232,7 +6936,7 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, int min_valid_off; if (t == BPF_WRITE || env->allow_uninit_stack) - min_valid_off = -MAX_BPF_STACK; + min_valid_off = -(int)env->stack_limit; else min_valid_off = -state->allocated_stack; @@ -6298,10 +7002,19 @@ static int check_stack_access_within_bounds( return err; } + /* + * Every stack access passes through here, including buffers passed to + * helpers and kfuncs and accesses into a caller's frame, so check the + * fastcall contract of the frame that owns the slots once for all of + * them. Zero-sized accesses touch no stack and keep the rewrite enabled. + */ + if (access_size) + check_fastcall_stack_contract(env, state, env->insn_idx, min_off); + /* Note that there is no stack access with offset zero, so the needed stack * size is -min_off, not -min_off+1. */ - return grow_stack_state(env, state, -min_off /* size */); + return bpf_grow_stack_state(env, state, -min_off /* size */); } static bool get_func_retval_range(struct bpf_prog *prog, @@ -6325,6 +7038,7 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, val); + scalar_step_add(dst_reg, dst_reg, &fake_reg); scalar32_min_max_add(dst_reg, &fake_reg); scalar_min_max_add(dst_reg, &fake_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off); @@ -6332,12 +7046,147 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +/* there is no support for callx in the interpreter */ +static int require_callx_jit(struct bpf_verifier_env *env) +{ + if (!env->prog->jit_requested) { + verbose(env, "JIT is required to use callx\n"); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_callx()) { + verbose(env, "JIT doesn't support callx\n"); + return -EOPNOTSUPP; + } + env->prog->jit_required = true; + return 0; +} + +static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int regno, int subprog) +{ + mark_reg_known_zero(env, regs, regno); + regs[regno].type = PTR_TO_FUNC; + regs[regno].subprogno = subprog; +} + +/* a read from a table of functions branches into that many states at most */ +#define BPF_MAX_FUNC_PTR_TARGETS 64 +/* and that many pointers at most are next to what is read at variable offset */ +#define BPF_MAX_FUNC_PTR_RANGE 4096 + +/* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */ +static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit) +{ + u64 o = min_off, next; + u32 cnt = 0; + + if (!tnum_in(offs, tnum_const(o))) { + next = tnum_step(offs, o); + if (next <= o) + return 0; + o = next; + } + while (o <= max_off && cnt < limit) { + cnt++; + next = tnum_step(offs, o); + if (next <= o) + break; + o = next; + } + return cnt; +} + +/* + * A read from a frozen read-only map that has pointers to functions, see + * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read + * is possible at, e.g. var_off is the stride of the elements of an array. + * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable + * and only pointers can be read, which is how an element of a table of + * functions is loaded, the verification continues with each of them. Other + * reads that overlap with a pointer are rejected, because their result is not + * known until the program is jitted. + * + * Return -ENOENT if there are no pointers to functions in the bytes that are read. + */ +static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int size, int value_regno) +{ + u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off; + struct tnum offs = tnum_add(reg->var_off, tnum_const(off)); + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_map *map = reg->map_ptr; + struct bpf_verifier_state *branch; + int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS]; + struct bpf_func_ptr *ptrs; + u32 i, cnt, n = 0; + u64 o, lo, hi; + int err; + + ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt); + if (!ptrs) + return -ENOENT; + if (cnt > BPF_MAX_FUNC_PTR_RANGE) + goto overlap; + + for (i = 0; i < cnt; i++) { + /* the offsets at which the read overlaps with the pointer */ + lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0; + lo = max(lo, min_off); + hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off); + for (o = lo; o <= hi; o++) { + if (!tnum_in(offs, tnum_const(o))) + continue; + if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0) + goto overlap; + if (n == BPF_MAX_FUNC_PTR_TARGETS) { + verbose(env, "read from map '%s' may yield more than %d pointers to functions\n", + map->name, BPF_MAX_FUNC_PTR_TARGETS); + return -E2BIG; + } + subprog = bpf_find_subprog(env, ptrs[i].xlated_off); + if (verifier_bug_if(subprog <= 0, env, + "no function at insn %u for map '%s' offset %u", + ptrs[i].xlated_off, map->name, ptrs[i].map_off)) + return -EFAULT; + targets[n++] = subprog; + } + } + /* pointers are next to what is read, e.g. the data of an array of vtables */ + if (!n) + return -ENOENT; + /* either every offset that the read is possible at is a pointer or none */ + if (count_read_offsets(offs, min_off, max_off, n + 1) != n) + goto overlap; + + /* the map has the offset of the function until the program is jitted */ + err = require_callx_jit(env); + if (err) + return err; + + for (i = 0; i < n - 1; i++) { + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, + env->cur_state->speculative); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno, + targets[i]); + } + mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]); + return 0; + +overlap: + verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n", + size, min_off, max_off, map->name); + return -EACCES; +} + static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int bpf_size, int value_regno, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); int size = bpf_size_to_bytes(bpf_size); struct bpf_map *map = reg->map_ptr; + int err; switch (map->map_type) { case BPF_MAP_TYPE_INSN_ARRAY: @@ -6355,13 +7204,18 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state break; } + if (env->func_ptr_cnt) { + err = check_func_ptr_read(env, reg, off, size, value_regno); + if (err != -ENOENT) + return err; + } + /* If map is read-only, track its contents as scalars. */ if (tnum_is_const(reg->var_off) && bpf_map_is_rdonly(map) && map->ops->map_direct_value_addr) { int map_off = off + reg->var_off.value; u64 val = 0; - int err; err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); if (err) @@ -6388,6 +7242,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6513,6 +7368,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; regs[value_regno].id = info.ref_id; + } else if (base_type(info.reg_type) == PTR_TO_MEM) { + regs[value_regno].mem_size = info.mem_size; } if (type_may_be_null(info.reg_type) && !regs[value_regno].id) regs[value_regno].id = ++env->id_gen; @@ -6594,7 +7451,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } err = check_buffer_access(env, reg, argno, off, size, false, - max_access); + max_access, t == BPF_WRITE); if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); @@ -6635,12 +7492,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); +/* + * Returns the register to check the access of the current insn with. + * For a number in a program with an arena that is 'arena'. + */ +static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, + struct bpf_reg_state *arena) +{ + if (!is_arena_scalar(env, regno)) + return cur_regs(env) + regno; + + memset(arena, 0, sizeof(*arena)); + arena->type = PTR_TO_ARENA; + env->insn_aux_data[env->insn_idx].arena_scalar = true; + return arena; +} + static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -6648,6 +7525,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -6669,16 +7547,28 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg_type = regs[insn->src_reg].type; + src_reg = mem_access_reg(env, insn->src_reg, &arena); + src_reg_type = src_reg->type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, - BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), + insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); + /* + * Paths through a kernel struct pointer and through other pointers + * may share the insn, see save_aux_ptr_type(). Record a load that did + * not produce an arena pointer from an __arena member, the only load + * through a kernel struct pointer that does, next to the one + * check_arena_member_load() records, so that arena_member_fixup() can + * tell when the paths disagree. + */ + if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && + regs[insn->dst_reg].type == PTR_TO_ARENA)) + env->insn_aux_data[env->insn_idx].non_arena_load = true; err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); @@ -6694,6 +7584,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -6715,11 +7606,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg_type = regs[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, - BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -6729,7 +7621,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state *dst_reg; + struct bpf_reg_state arena, *dst_reg; int load_reg; int err; @@ -6774,6 +7666,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } + /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -6782,8 +7677,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } - dst_reg = cur_regs(env) + insn->dst_reg; - /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -6910,8 +7803,11 @@ static int check_stack_range_initialized( * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; - /* The call will initialize the memory; uninitialized stack allowed */ - bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); + u32 arg_slot = arg_slot_from_argno(argno); + bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS && + (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool raw_mode = uninit && env->allow_uninit_stack && + !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot)); access_size = abs(access_size); @@ -6940,11 +7836,9 @@ static int check_stack_range_initialized( reg_arg_name(env, argno), tn_buf); return -EACCES; } - /* Only initialized buffer on stack is allowed to be accessed - * with variable offset. With uninitialized buffer it's hard to - * guarantee that whole memory is marked as initialized on - * helper return since specific bounds are unknown what may - * cause uninitialized stack leaking. + /* + * The call may touch any byte in the possible range, but does not + * definitely initialize all of it. Fall back to ordinary stack checks. */ raw_mode = false; @@ -6952,12 +7846,14 @@ static int check_stack_range_initialized( max_off = reg_smax(reg) + off; } + /* Unprivileged outputs retain each byte's initialization state. */ if (raw_mode) { - meta->arg_raw_mem.size = access_size; + meta->arg_raw_mem.size[arg_slot] = access_size; return 0; } for (i = min_off; i < max_off + access_size; i++) { + struct bpf_stack_state *ss; u8 *stype; slot = -i - 1; @@ -6967,32 +7863,41 @@ static int check_stack_range_initialized( return -EFAULT; } - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || - (*stype == STACK_INVALID && env->allow_uninit_stack)) { - if (clobber) { + (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) { + if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) { /* helper can write anything into the stack */ *stype = STACK_MISC; } goto mark; } - if (bpf_is_spilled_reg(&state->stack[spi]) && - (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || + /* + * Only the bytes marked STACK_SPILL hold the spilled register. + * The rest of a narrowly spilled slot keeps its previous type + * and must be initialized on its own. + */ + if (*stype == STACK_SPILL && + (ss->spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { if (clobber) { - __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); + __mark_reg_unknown(env, &ss->spilled_ptr); for (j = 0; j < BPF_REG_SIZE; j++) - scrub_spilled_slot(&state->stack[spi].slot_type[j]); + scrub_spilled_slot(&ss->slot_type[j]); } goto mark; } if (*stype == STACK_POISON) { - if (allow_poison) + if (allow_poison) { + if (uninit && env->allow_uninit_stack) + *stype = STACK_MISC; goto mark; + } verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { @@ -7026,6 +7931,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: + if (!may_access_direct_pkt_data(env, meta, access_type)) { + verbose(env, "function access to the packet is not allowed\n"); + return -EACCES; + } return check_packet_access(env, reg, argno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: @@ -7066,7 +7975,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } return check_buffer_access(env, reg, argno, 0, access_size, zero_size_allowed, - max_access); + max_access, access_type == BPF_WRITE); case PTR_TO_STACK: return check_stack_range_initialized( env, reg, @@ -7136,13 +8045,13 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, */ meta->msize_max_value = reg_umax(size_reg); - /* The register is SCALAR_VALUE; the access check happens using - * its boundaries. For unprivileged variable accesses, disable - * raw mode so that the program is required to initialize all - * the memory that the helper could just partially fill up. + /* + * Check variable ranges byte by byte instead of using raw mode. Keep the + * MEM_UNINIT annotation so invalid bytes are accepted without marking them + * initialized when the caller cannot read uninitialized stack memory. */ if (!tnum_is_const(size_reg->var_off)) - meta = NULL; + meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno)); if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -7185,6 +8094,11 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, return err; } +static bool is_subprog(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && !meta->func_id; +} + static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, enum bpf_access_type access_type, struct bpf_call_arg_meta *meta, bool *known_memory) @@ -7203,10 +8117,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg } /* - * Only a global subprog (meta == NULL) may read poisoned stack slots: + * Only a global subprog may read poisoned stack slots: * its static stack liveness proved the callee body skips them. */ - size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; + size = (is_subprog(meta) && + base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; if (access_type & BPF_READ) err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, @@ -7498,6 +8413,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_ case BPF_TASK_WORK: field_off = map->record->task_work_off; break; + case BPF_RCU_HEAD: + field_off = map->record->rcu_head_off; + break; case BPF_WORKQUEUE: field_off = map->record->wq_off; break; @@ -7586,11 +8504,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u /* * Validate dynptr arguments for helper, kfunc and subprog. * - * @dynptr is both input and output. It is populated when the argument is - * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed) - * and consumed when the argument is expecting to be an initialized dynptr. - * @parent_id is used to track the referenced parent object (e.g., file or skb in - * qdisc program) when constructing a dynptr. + * @meta carries the dynptr and referenced-object state. The dynptr is populated + * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that + * will be constructed) and consumed when the argument is expected to be an + * initialized dynptr. The reference tracks the parent object (e.g., file or skb + * in qdisc program) when constructing a dynptr. * * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. @@ -7607,24 +8525,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, const char *call_name, - enum bpf_arg_type arg_type, - struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) + argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta) { int spi, err = 0; - if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { - verbose(env, - "%s expected pointer to stack or const struct bpf_dynptr\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, - "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", - "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", - reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - /* MEM_UNINIT - Points to memory that is an appropriate candidate for * constructing a mutable bpf_dynptr object. * @@ -7646,7 +8551,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Dynptr has to be an uninitialized dynptr\n"); bpf_diag_res( env, insn_idx, "dynptr is already initialized", - "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7659,7 +8564,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } - err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr); + err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, + &meta->ref_obj, &meta->dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { @@ -7689,7 +8595,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Expected a dynptr of type %s as %s\n", dynptr_type_str(expected_type), reg_arg_name(env, argno)); bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, + env, insn_idx, argno, meta->func_name, "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", dynptr_type_str(actual_type), dynptr_type_str(expected_type)); @@ -7705,14 +8611,12 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; } - if (dynptr) { - dynptr->type = reg->dynptr.type; - dynptr->id = reg->id; - dynptr->parent_id = reg->parent_id; - } + meta->dynptr.type = reg->dynptr.type; + meta->dynptr.id = reg->id; + meta->dynptr.parent_id = reg->parent_id; } return err; } @@ -7745,12 +8649,12 @@ static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, return btf_param_match_suffix(meta->btf, arg, "__iter"); } -static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, +static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, int insn_idx, struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; - u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id; if (reg->type != PTR_TO_STACK) { @@ -7770,14 +8674,14 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * * to any kfunc, if arg has "__iter" suffix, we need to be a bit more * conservative here. */ - btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx); + btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg); if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); bpf_diag_call_arg( env, insn_idx, argno, meta->func_name, - "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", - "Pass the exact iterator state type expected by this kfunc."); + "the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this function."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7840,7 +8744,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; - update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr); + update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); @@ -7917,16 +8821,15 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, &fold->regs[i], &fcur->regs[i]); - num_slots = min(fold->allocated_stack / BPF_REG_SIZE, - fcur->allocated_stack / BPF_REG_SIZE); + num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur)); for (i = 0; i < num_slots; i++) { - if (!bpf_is_spilled_reg(&fold->stack[i]) || - !bpf_is_spilled_reg(&fcur->stack[i])) + struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i); + struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i); + + if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss)) continue; - maybe_widen_reg(env, - &fold->stack[i].spilled_ptr, - &fcur->stack[i].spilled_ptr); + maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr); } } return 0; @@ -7938,7 +8841,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; - return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr; + return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr; } /* process_iter_next_call() is called when verifier gets to iterator's next @@ -8101,9 +9004,52 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type) return base_type(type) == ARG_PTR_TO_DYNPTR; } +/* + * An argument that only ever takes a scalar, so a zero register passed to it + * is a value rather than a NULL pointer. + */ +static bool arg_type_is_scalar(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + case ARG_CONST_MEM_SIZE: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return true; + default: + return false; + } +} + +/* + * A helper has no BTF and a nonzero function ID. A kfunc has both, while a + * BPF subprogram has BTF and a zero function ID. + */ +static bool is_helper(const struct bpf_call_arg_meta *meta) +{ + return !meta->btf && meta->func_id; +} + +static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id) +{ + return is_helper(meta) && meta->func_id == func_id; +} + +static bool is_kfunc(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && meta->func_id; +} + +static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) +{ + return is_kfunc(meta) && meta->func_id == btf_id; +} + static int resolve_map_arg_type(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, - enum bpf_arg_type *arg_type) + const struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type) { if (!meta->map.ptr) { /* kernel subsystem misconfigured verifier */ @@ -8122,7 +9068,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, } break; case BPF_MAP_TYPE_BLOOM_FILTER: - if (meta->func_id == BPF_FUNC_map_peek_elem) + if (is_helper_call(meta, BPF_FUNC_map_peek_elem)) *arg_type = ARG_PTR_TO_MAP_VALUE; break; default: @@ -8131,6 +9077,49 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, return 0; } +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type, u32 *arg_size); +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx); +static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, + const struct btf_param *arg); +static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, + const struct btf_param *arg, + const char *name); +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta); +static bool is_bpf_obj_drop_kfunc(u32 func_id); +static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id); +static bool is_bpf_rbtree_add_kfunc(u32 func_id); +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta); +static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); +static int process_irq_flag(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env); + struct bpf_reg_types { const enum bpf_reg_type types[10]; u32 *btf_id; @@ -8174,7 +9163,7 @@ static const struct bpf_reg_types mem_types = { }, }; -static const struct bpf_reg_types spin_lock_types = { +static const struct bpf_reg_types map_value_or_alloc_obj_types = { .types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, @@ -8203,7 +9192,33 @@ static const struct bpf_reg_types percpu_btf_ptr_types = { static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; -static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types arena_types = { + .types = { + PTR_TO_ARENA, + SCALAR_VALUE, + } +}; +static const struct bpf_reg_types ctx_out_types = { + .types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED }, +}; + +static const struct bpf_reg_types alloc_obj_drop_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU, + } +}; + +static const struct bpf_reg_types alloc_obj_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, + } +}; + static const struct bpf_reg_types kptr_xchg_dest_types = { .types = { PTR_TO_MAP_VALUE, @@ -8234,16 +9249,32 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { #endif [ARG_PTR_TO_SOCKET] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, - [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, + [ARG_PTR_TO_SPIN_LOCK] = &map_value_or_alloc_obj_types, [ARG_PTR_TO_MEM] = &mem_types, [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, [ARG_PTR_TO_STACK] = &stack_ptr_types, [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, - [ARG_PTR_TO_TIMER] = &timer_types, + [ARG_PTR_TO_TIMER] = &map_value_types, [ARG_KPTR_XCHG_DEST] = &kptr_xchg_dest_types, [ARG_PTR_TO_DYNPTR] = &dynptr_types, + [ARG_CONST_SCALAR] = &scalar_types, + [ARG_CONST_MEM_SIZE] = &scalar_types, + [ARG_PTR_TO_ALLOC_BTF_ID] = &alloc_obj_drop_types, + [ARG_PTR_TO_REFCOUNTED_KPTR] = &alloc_obj_types, + [ARG_PTR_TO_ITER] = &stack_ptr_types, + [ARG_PTR_TO_LIST_HEAD] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_LIST_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RB_ROOT] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_RB_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RES_SPIN_LOCK] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_WORKQUEUE] = &map_value_types, + [ARG_PTR_TO_TASK_WORK] = &map_value_types, + [ARG_PTR_TO_RCU_HEAD] = &map_value_types, + [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, + [ARG_PTR_TO_ARENA] = &arena_types, + [ARG_PTR_TO_CTX_OUT] = &ctx_out_types, }; static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, @@ -8283,6 +9314,71 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); } +static int check_func_arg_nullability(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) || + (!bpf_register_is_null(reg) && !type_may_be_null(reg->type))) + return 0; + + if (meta->btf) { + u32 arg_btf_id; + + arg_btf_id = btf_params(meta->func_proto)[arg].type; + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id)); + } + + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Add a NULL check and make the call only on the non-NULL path.", + "the pointer may be NULL, but this call requires a non-NULL %s", + expected_type); + return -EACCES; +} + +static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (!arg_type_is_release(arg_type)) + return 0; + + if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) || + bpf_register_is_null(reg)) + return 0; + + verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n", + meta->func_name, reg_arg_name(env, argno)); + + if (is_kfunc(meta)) { + const struct btf_param *btf_arg; + const struct btf_type *t; + u32 ref_id; + + btf_arg = &btf_params(meta->func_proto)[arg]; + ref_id = btf_arg->type; + t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL); + if (btf_type_is_ptr(t)) + btf_type_skip_modifiers(meta->btf, t->type, &ref_id); + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, ref_id)); + } + + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.", + expected_type), + "release functions require a value that owns a live resource returned by a matching acquire function"); + return -EINVAL; +} + static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, const enum bpf_reg_type *types, int count) { @@ -8304,19 +9400,21 @@ static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, } static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta, const char *call_name) + enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; const char *actual, *accepted; - int i, j, err; + int i, j; compatible = compatible_reg_types[base_type(arg_type)]; if (!compatible) { verifier_bug(env, "unsupported arg type %d", arg_type); return -EFAULT; } + if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID && + (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)])) + goto found; /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY @@ -8336,9 +9434,14 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~PTR_MAYBE_NULL; if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK; + /* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */ + if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && + type_is_ptr_alloc_obj(type)) + type = PTR_TO_MEM; /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ - if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) { + if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) && + reg_from_argno(argno) == BPF_REG_2) { type &= ~MEM_ALLOC; type &= ~MEM_PERCPU; } @@ -8358,115 +9461,13 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); accepted = bpf_diag_expected_reg_types(env, compatible->types, i); - bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, - "Pass a value with one of the accepted pointer or scalar types for this call.", + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)), "it has type %s, but this argument accepts %s", actual, accepted); return -EACCES; found: - if (base_type(reg->type) != PTR_TO_BTF_ID) - return 0; - - if (compatible == &mem_types) { - if (!(arg_type & MEM_RDONLY)) { - verbose(env, - "%s() may write into memory pointed by %s type=%s\n", - func_id_name(meta->func_id), - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - return -EACCES; - } - return 0; - } - - switch ((int)reg->type) { - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_TRUSTED: - case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | MEM_RCU: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: - { - /* For bpf_sk_release, it needs to match against first member - * 'struct sock_common', hence make an exception for it. This - * allows bpf_sk_release to work for multiple socket types. - */ - bool strict_type_match = arg_type_is_release(arg_type) && - meta->func_id != BPF_FUNC_sk_release; - - if (type_may_be_null(reg->type) && - (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { - verbose(env, "Possibly NULL pointer passed to helper %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, env->insn_idx, argno, call_name, - "the pointer may be NULL, but this call requires a non-NULL pointer", - "Add a NULL check and make the call only on the non-NULL path."); - return -EACCES; - } - - if (!arg_btf_id) { - if (!compatible->btf_id) { - verifier_bug(env, "missing arg compatible BTF ID"); - return -EFAULT; - } - arg_btf_id = compatible->btf_id; - } - - if (meta->func_id == BPF_FUNC_kptr_xchg) { - if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) - return -EACCES; - } else { - if (arg_btf_id == BPF_PTR_POISON) { - verbose(env, "verifier internal error:"); - verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", - reg_arg_name(env, argno)); - return -EACCES; - } - - err = __check_ptr_off_reg(env, reg, argno, true); - if (err) - return err; - - if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, - reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match, !type_is_alloc(reg->type))) { - verbose(env, "%s is of type %s but %s is expected\n", - reg_arg_name(env, argno), - btf_type_name(reg->btf, reg->btf_id), - btf_type_name(btf_vmlinux, *arg_btf_id)); - return -EACCES; - } - } - break; - } - case PTR_TO_BTF_ID | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: - if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && - meta->func_id != BPF_FUNC_kptr_xchg) { - verifier_bug(env, "unimplemented handling of MEM_ALLOC"); - return -EFAULT; - } - /* Check if local kptr in src arg matches kptr in dst arg */ - if (meta->func_id == BPF_FUNC_kptr_xchg) { - int regno = reg_from_argno(argno); - - if (regno == BPF_REG_2 && - map_kptr_match_type(env, meta->kptr_field, reg, regno)) - return -EACCES; - } - break; - case PTR_TO_BTF_ID | MEM_PERCPU: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: - case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: - /* Handled by helper specific checks */ - break; - default: - verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match"); - return -EFAULT; - } return 0; } @@ -8487,10 +9488,9 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int __check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - bool btf_id_fixed_off_ok) +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) { u32 type = reg->type; @@ -8546,12 +9546,15 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, - * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero unless the caller requires otherwise. - * var_off always must be 0 for PTR_TO_BTF_ID, hence we still - * need to do checks instead of returning. + * its fixed offset must be 0. bpf_refcount_acquire() returns the + * pointer it was given while incrementing the refcount at the + * refcount field offset, so it needs a zero offset too. In the + * other cases, fixed offset can be non-zero. var_off always must + * be 0 for PTR_TO_BTF_ID, hence we still need to do checks + * instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); + return __check_ptr_off_reg(env, reg, argno, + base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8559,7 +9562,7 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, * otherwise we may get modified ctx in tail called programs and * global subprogs (that may act as extension prog hooks). */ - if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) + if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) return 0; fallthrough; default: @@ -8567,13 +9570,6 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, } } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) -{ - return __check_func_arg_reg_off(env, reg, argno, arg_type, true); -} - static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -8582,6 +9578,7 @@ static int check_arg_const_str(struct bpf_verifier_env *env, int map_off; u64 map_addr; char *str_ptr; + u32 cnt; if (reg->type != PTR_TO_MAP_VALUE) return -EINVAL; @@ -8631,6 +9628,11 @@ static int check_arg_const_str(struct bpf_verifier_env *env, verbose(env, "string is not zero-terminated\n"); return -EINVAL; } + /* the bytes of a pointer to a function are not known until the program is jitted */ + if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) { + verbose(env, "string overlaps with a pointer to a function\n"); + return -EACCES; + } return 0; } @@ -8641,6 +9643,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, s64 *value) { struct bpf_func_state *state = bpf_func(env, key); + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int slot, spi, off; int spill_size = 0; @@ -8660,7 +9663,8 @@ static int get_constant_map_key(struct bpf_verifier_env *env, slot = -stack_off - 1; spi = slot / BPF_REG_SIZE; off = slot % BPF_REG_SIZE; - stype = state->stack[spi].slot_type; + ss = bpf_stack_slot(state, spi); + stype = ss->slot_type; /* First handle precisely tracked STACK_ZERO */ for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) @@ -8671,14 +9675,14 @@ static int get_constant_map_key(struct bpf_verifier_env *env, } /* Check that stack contains a scalar spill of expected size */ - if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) + if (!bpf_is_spilled_scalar_reg(ss)) return -EOPNOTSUPP; for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) spill_size++; if (spill_size != key_size) return -EOPNOTSUPP; - reg = &state->stack[spi].spilled_ptr; + reg = &ss->spilled_ptr; if (!tnum_is_const(reg->var_off)) /* Stack value not statically known */ return -EOPNOTSUPP; @@ -8722,6 +9726,8 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat obj_name = "timer"; else if (rec->task_work_off >= 0) obj_name = "bpf_task_work"; + else if (rec->rcu_head_off >= 0) + obj_name = "bpf_rcu_head"; verbose(env, "%s pointer in %s map_uid=%d ", obj_name, reg_arg_name(env, obj_argno), meta->map.uid); @@ -8735,65 +9741,89 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat return 0; } -static int check_func_arg(struct bpf_verifier_env *env, u32 arg, +/* + * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output + * whose incoming contents are not read. + */ +static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type) +{ + if (!(arg_type & MEM_WRITE)) + return BPF_READ; + if (arg_type & MEM_UNINIT) + return BPF_WRITE; + return BPF_READ | BPF_WRITE; +} + +static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, struct bpf_call_arg_meta *meta, int insn_idx) { + const struct btf_param *btf_arg = is_kfunc(meta) ? + &btf_params(meta->func_proto)[arg] : NULL; const struct bpf_func_proto *fn = meta->fn; - u32 regno = BPF_REG_1 + arg; - struct bpf_reg_state *reg = reg_state(env, regno); + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *regs = cur_regs(env); + argno_t argno = argno_from_arg(slot + 1); + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_reg(regno); - enum bpf_reg_type type = reg->type; - u32 *arg_btf_id = NULL; + int regno = reg_from_argno(argno); + u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; u32 key_size; int err = 0; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_PTR_TO_PROG_AUX) { + cur_aux(env)->arg_prog = regno; + return 0; + } + + if (arg_type == ARG_IGNORE) return 0; - err = check_reg_arg(env, regno, SRC_OP); - if (err) - return err; + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + /* Preserve the legacy helper behavior for privileged pointer leaks. */ if (arg_type == ARG_ANYTHING) { - if (is_pointer_value(env, regno)) { - verbose(env, "R%d leaks addr into helper function\n", - regno); + if (__is_pointer_value(env->allow_ptr_leaks, reg)) { + verbose(env, "%s leaks addr into helper function\n", + reg_arg_name(env, argno)); return -EACCES; } return 0; } - if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, fn, BPF_READ)) { - verbose(env, "helper access to the packet is not allowed\n"); + err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size); + if (err) + return err; + + /* + * A slice of a packet is packet data under another name. A helper which + * may reallocate the packet must not go on reading from the old one. + */ + if (is_helper(meta) && reg_is_dynptr_slice_pkt(reg) && + bpf_helper_changes_pkt_data(meta->func_id)) { + verbose(env, "function access to the packet is not allowed\n"); return -EACCES; } - if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { - err = resolve_map_arg_type(env, meta, &arg_type); - if (err) - return err; - } + if (arg_type_is_raw_mem(arg_type)) + meta->arg_raw_mem.mask |= BIT(slot); if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { - /* A NULL register has a SCALAR_VALUE type, so skip - * type checking. - */ - err = mark_chain_precision(env, regno); + err = mark_arg_precision(env, argno); if (err) return err; - goto skip_type_check; + return 0; } - /* arg_btf_id and arg_size are in a union. */ - if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || - base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) - arg_btf_id = fn->arg_btf_id[arg]; + err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx); + if (err) + return err; - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, - func_id_name(meta->func_id)); + err = check_reg_type(env, reg, argno, arg_type, meta); if (err) return err; @@ -8801,22 +9831,27 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; -skip_type_check: - if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_id_name(meta->func_id), reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, insn_idx, argno, func_id_name(meta->func_id), - "release helpers require a value that owns a live resource returned by a matching acquire helper", - "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); - return -EINVAL; - } + err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx); + if (err) + return err; if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); switch (base_type(arg_type)) { + case ARG_CONST_SCALAR: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + break; + case ARG_SCALAR: + break; case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ err = process_map_ptr_arg(env, reg, argno, meta); @@ -8838,7 +9873,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta, NULL); if (err) return err; @@ -8866,21 +9901,86 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EFAULT; } - /* - * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads - * the value buffer as an input rather than filling it. - */ - if (meta->func_id == BPF_FUNC_map_peek_elem && - meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) - meta->arg_raw_mem.regno = 0; - err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta, NULL); break; + case ARG_PTR_TO_BTF_ID: + case ARG_PTR_TO_BTF_ID_SOCK_COMMON: + { + const u32 *arg_btf_id = fn->arg_btf_id[arg]; + const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux; + enum bpf_reg_type reg_type; + + if (!meta->btf) { + const struct bpf_reg_types *compatible; + + if (base_type(reg->type) != PTR_TO_BTF_ID) + break; + + if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) + return map_kptr_match_type(env, meta->kptr_field, reg, regno) ? + -EACCES : 0; + + if (!arg_btf_id) { + compatible = compatible_reg_types[base_type(arg_type)]; + if (!compatible->btf_id) { + verifier_bug(env, "missing arg compatible BTF ID"); + return -EFAULT; + } + arg_btf_id = compatible->btf_id; + } + if (arg_btf_id == BPF_PTR_POISON) { + verbose(env, "verifier internal error:"); + verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", + reg_arg_name(env, argno)); + return -EACCES; + } + } + + reg_type = reg->type & ~(arg_type & PTR_MAYBE_NULL); + if ((arg_type & PTR_TRUSTED) && + (!is_trusted_reg_type(env, reg, reg_type) || + bpf_type_has_unsafe_modifiers(reg_type))) { + if (!(arg_type & MEM_RCU)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.", + "the function requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Use this function with a pointer that is valid in an RCU read lock region.", + "the function requires an RCU-protected pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + } + + err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf, + *arg_btf_id, meta, insn_idx); + if (err < 0) + return err; + break; + } case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { - verbose(env, "Helper has invalid btf_id in R%d\n", regno); + verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno)); return -EACCES; } meta->ret_btf = reg->btf; @@ -8891,11 +9991,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); return -EACCES; } - if (meta->func_id == BPF_FUNC_spin_lock) { + if (is_helper_call(meta, BPF_FUNC_spin_lock)) { err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; - } else if (meta->func_id == BPF_FUNC_spin_unlock) { + } else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) { err = process_spin_lock(env, reg, argno, 0); if (err) return err; @@ -8909,45 +10009,283 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; break; + case ARG_PTR_TO_CTX: + if (is_bpf_cast_to_kern_ctx_kfunc(meta)) { + err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); + if (err < 0) + return -EINVAL; + meta->ret_btf_id = err; + } + break; + case ARG_PTR_TO_ARENA: + break; + case ARG_PTR_TO_ALLOC_BTF_ID: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { + if (!is_bpf_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { + if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_percpu_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (meta->btf == btf_vmlinux) { + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + } + break; case ARG_PTR_TO_FUNC: meta->subprogno = reg->subprogno; break; case ARG_PTR_TO_MEM: + { + enum bpf_access_type access_type; + bool known_memory; + /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - if (arg_type & MEM_FIXED_SIZE) { - err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); - if (err) - return err; - if (arg_type & MEM_ALIGNED) - err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); + if (!(arg_type & MEM_FIXED_SIZE)) + break; + + access_type = func_arg_access_type(arg_type); + + err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); + if (err < 0) { + if (known_memory) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass memory with at least the required number of accessible bytes and suitable read or write access.", + "the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the function expects %u bytes of memory, but it is %s and not verifier-known memory", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + return err; + } + /* + * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting + * bounds tracking information. + */ + if (is_subprog(meta) && meta->subprog_may_change_pkt && + (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) { + verbose(env, + "cannot pass packet pointer %s to %s(): function may change packet data\n", + reg_arg_name(env, argno), meta->func_name); + return -EINVAL; + } + if (arg_type & MEM_ALIGNED) + err = check_ptr_alignment(env, reg, 0, arg_size, true); + break; + } + case ARG_CONST_MEM_SIZE: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + fallthrough; + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot); + argno_t buff_argno = argno_from_arg(prev_slot + 1); + enum bpf_mem_size_failure failure; + const char *buff_arg, *size_arg; + bool zero_size_allowed; + u32 access_type; + + if (meta->btf && bpf_register_is_null(buff_reg)) + break; + + access_type = func_arg_access_type(fn->arg_type[arg - 1]); + + zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; + + err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno, + access_type, zero_size_allowed, meta, &failure); + if (!err) + break; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(reg) < 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(reg)); + else if (!zero_size_allowed && reg_umin(reg) == 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Ensure the memory size is non-zero before this call.", + "the memory size in %s may be zero, but the function requires a non-zero size", + size_arg); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ); } break; - case ARG_MEM_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta, NULL); - break; - case ARG_MEM_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - true, meta, NULL); - break; - case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), - arg_type, &meta->ref_obj, &meta->dynptr); + } + case ARG_PTR_TO_DYNPTR: { + if (is_bpf_dynptr_clone_kfunc(meta) && + (arg_type & MEM_UNINIT)) { + enum bpf_dynptr_type parent_type = meta->dynptr.type; + + if (parent_type == BPF_DYNPTR_TYPE_INVALID) { + verifier_bug(env, "no dynptr type for parent of clone"); + return -EFAULT; + } + + arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); + } + + err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); if (err) return err; + if (!arg && is_kfunc_dynptr_write(meta)) + meta->dynptr_write_type = meta->dynptr.type; + break; + } + case ARG_PTR_TO_ITER: + if (is_bpf_iter_css_task_new_kfunc(meta) && + !check_css_task_iter_allowlist(env)) { + verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); + return -EINVAL; + } + err = process_iter_arg(env, reg, arg, argno, insn_idx, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_HEAD: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_ROOT: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_NODE: + if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) && + type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_NODE: + if (is_bpf_rbtree_add_kfunc(meta->func_id)) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } else { + if (!type_is_non_owning_ref(reg->type) && + !reg_is_referenced(env, reg)) { + verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", + meta->func_name); + return -EINVAL; + } + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "%s not allowed in rbtree cb\n", meta->func_name); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); + if (err < 0) + return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg, + "rdonly_buf_size")) + meta->r0_rdonly = true; err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (err) + if (err < 0) { + if (meta->btf && err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a verifier-known constant size for this function's buffer argument.", + "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); return err; + } break; + case ARG_PTR_TO_REFCOUNTED_KPTR: + { + struct btf_record *rec; + + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) + meta->arg_owning_ref = true; + + rec = reg_btf_record(reg); + if (!rec) { + verifier_bug(env, "Couldn't find btf_record"); + return -EFAULT; + } + + if (rec->refcount_off < 0) { + verbose(env, "%s doesn't point to a type with bpf_refcount field\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + break; + } case ARG_PTR_TO_CONST_STR: { err = check_arg_const_str(env, reg, argno); @@ -8955,6 +10293,50 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; break; } + case ARG_PTR_TO_WORKQUEUE: + err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_TASK_WORK: + err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RCU_HEAD: + err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_IRQ_FLAG: + err = process_irq_flag(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_CTX_OUT: + if (reg->mem_size != arg_size) { + verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n", + reg_arg_name(env, argno), arg_size, reg->mem_size); + return -EINVAL; + } + break; + case ARG_PTR_TO_RES_SPIN_LOCK: + { + int flags; + + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + + flags = get_bpf_res_spin_lock_kfunc_flags(meta); + if (!flags) + return -EFAULT; + err = process_spin_lock(env, reg, argno, flags); + if (err < 0) + return err; + break; + } case ARG_KPTR_XCHG_DEST: err = process_kptr_func(env, regno, meta); if (err) @@ -8965,6 +10347,139 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } +/* + * The slots a parameter takes, from its BTF type. This has to agree with + * btf_func_model_arg_slots(), which answers the same from the size the + * func model recorded, or the verifier would check an argument at a slot + * the JIT does not place it at. + */ +static u32 btf_arg_slots(const struct btf_type *t) +{ + if (btf_type_is_int(t) || btf_type_is_struct(t)) + return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + return 1; +} + +static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; + + for (i = 0; i < nargs; i++) + slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + + return slots_used; +} + +/* The argument slot @slot holds an eightbyte of a by-value argument. */ +static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller, + u32 slot, struct bpf_call_arg_meta *meta) +{ + struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot); + argno_t argno = argno_from_arg(slot + 1); + int regno = reg_from_argno(argno); + int err; + + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + + return check_reg_type(env, reg, argno, ARG_SCALAR, meta); +} + +static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + u32 slot = 0, prev_slot = 0, proto_slots, nslots; + struct bpf_func_state *caller = cur_func(env); + const struct btf_param *args = NULL; + u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; + int err; + + if (meta->btf) { + args = btf_params(meta->func_proto); + nargs = btf_type_vlen(meta->func_proto); + } + + /* + * A by-value argument of more than one eightbyte takes a slot per + * eightbyte, so the slots a call occupies are no longer its parameter + * count. Only a proto whose parameters take a slot each can name the + * argument a stack slot belongs to. + */ + proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs; + + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { + err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, + meta->btf, proto_slots == nargs ? args : NULL); + if (err) + return err; + } + + for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) { + const struct btf_type *t; + u32 k; + + nslots = 1; + if (args) { + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); + nslots = btf_arg_slots(t); + } + + if (meta->fn->arg_type[arg] == ARG_UNUSED) + break; + err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx); + if (err) + return err; + + /* + * check_func_arg() took the first slot. A parameter of more + * than one eightbyte is always a scalar, so every slot it + * takes beyond the first holds one too. + */ + for (k = 1; k < nslots; k++) { + err = check_arg_extra_slot(env, caller, slot + k, meta); + if (err) + return err; + } + } + + return 0; +} + +static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + u32 slot; + int i, err; + + /* + * Validate every argument before initializing outputs: an input argument + * may alias an output buffer. Use the normal stack-write checks to discard + * stale spills and preserve the rules for special stack objects. + */ + for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) { + struct bpf_reg_state *reg; + argno_t argno = argno_from_arg(slot + 1); + + if (!meta->arg_raw_mem.size[slot]) + continue; + reg = get_func_arg_reg(caller, cur_regs(env), slot); + + for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) { + err = check_mem_access(env, insn_idx, reg, argno, i, BPF_B, + BPF_WRITE, -1, false, false); + if (err) + return err; + } + } + + return 0; +} + static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; @@ -9004,12 +10519,71 @@ bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env) bpf_jit_supports_subprog_tailcalls(); } +/* + * Is rcu_read_lock_held() true at run time? Unlike for in_rcu_cs(), disabled + * preemption or interrupts will not do. + */ +static bool has_rcu_read_lock(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + + if (state->active_rcu_locks || in_rcu_callback(env)) + return true; + if (in_sleepable(env)) + return false; + return !env->no_rcu_read_lock || state->frame[0]->in_async_callback_fn; +} + +/* May the map be used outside of an RCU read side critical section? */ +static bool map_needs_no_rcu(const struct bpf_map *map) +{ + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_RHASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_PERCPU_ARRAY: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_LPM_TRIE: + case BPF_MAP_TYPE_ARRAY_OF_MAPS: + case BPF_MAP_TYPE_HASH_OF_MAPS: + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + case BPF_MAP_TYPE_INODE_STORAGE: + case BPF_MAP_TYPE_SK_STORAGE: + case BPF_MAP_TYPE_TASK_STORAGE: + case BPF_MAP_TYPE_CGRP_STORAGE: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_ARENA: + case BPF_MAP_TYPE_INSN_ARRAY: + case BPF_MAP_TYPE_PROG_ARRAY: + case BPF_MAP_TYPE_BLOOM_FILTER: + return true; + default: + return false; + } +} + static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func_id) { if (!map) return 0; + /* + * check_map_prog_compatibility() asks the program. That misses the + * sleepable callback of another program, and an inner map. They may + * use any map under the RCU read lock. Disabled preemption or + * interrupts will not do: the maps ask for rcu_read_lock_held(). + */ + if (!has_rcu_read_lock(env) && !map_needs_no_rcu(map)) { + verbose(env, "map '%s' needs bpf_rcu_read_lock() in sleepable context\n", + map->name); + return -EINVAL; + } + /* We need a two way check, first is from map perspective ... */ switch (map->map_type) { case BPF_MAP_TYPE_PROG_ARRAY: @@ -9257,23 +10831,6 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) - break; - if (!arg_type_is_raw_mem(fn->arg_type[i])) - continue; - if (meta->arg_raw_mem.regno) - return false; - meta->arg_raw_mem.regno = i + 1; - } - - return true; -} - static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) { bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; @@ -9312,7 +10869,7 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; @@ -9335,7 +10892,7 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; @@ -9353,7 +10910,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) @@ -9365,9 +10922,41 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ return true; } -static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) +static bool check_arg_prog_aux(struct bpf_verifier_env *env, + const struct bpf_func_proto *proto) { - return check_raw_mode_ok(fn, meta) && + bool seen = false; + argno_t argno; + u32 i; + + for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) { + if (proto->arg_type[i] == ARG_UNUSED) + break; + if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX) + continue; + + if (seen) { + verifier_bug(env, "Only 1 prog->aux argument supported"); + return false; + } + + argno = argno_from_arg(i + 1); + if (reg_from_argno(argno) < 0) { + verbose(env, "%s prog->aux cannot be a stack argument\n", + reg_arg_name(env, argno)); + return false; + } + + seen = true; + } + + return true; +} + +static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn, + struct bpf_call_arg_meta *meta) +{ + return check_arg_prog_aux(env, fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -9393,6 +10982,20 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) })); } +static bool holds_pkt_pointers(struct bpf_verifier_env *env) +{ + struct bpf_func_state *state; + struct bpf_reg_state *reg; + bool ret = false; + + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + ret = true; + })); + + return ret; +} + enum { AT_PKT_END = -1, BEYOND_PKT_END = -2, @@ -9587,7 +11190,8 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); - reg->id = 0; + if (!type_may_be_null(reg->type)) + reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; bpf_diag_mod_end(env); @@ -9679,23 +11283,57 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls return err; } -static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, - const struct btf *btf, - struct bpf_reg_state *regs) +static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf, + const struct btf_type *func_proto, struct bpf_func_proto *proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 arg, slot = 0; + + memset(proto, 0, sizeof(*proto)); + for (arg = 0; arg < btf_type_vlen(func_proto); arg++) { + enum bpf_arg_type arg_type = sub->args[slot].arg_type; + const struct btf_type *t; + + if (arg_type & PTR_UNTRUSTED) { + /* + * An __arg_untrusted argument accepts any caller value. The + * callee treats it as read-only and uses probe-read instructions + * to protect against invalid memory access. + */ + arg_type = ARG_IGNORE; + } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { + arg_type |= PTR_MAYBE_NULL; + } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { + proto->arg_size[arg] = sub->args[slot].mem_size; + arg_type |= MEM_FIXED_SIZE | MEM_WRITE; + } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + proto->arg_btf_id[arg] = &sub->args[slot].btf_id; + } + proto->arg_type[arg] = arg_type; + t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + slot += btf_arg_slots(t); + } +} + +static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf) { struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); - struct bpf_verifier_log *log = &env->log; - struct ref_obj_desc ref_obj = {}; - const struct btf_param *args; const struct btf_type *func, *func_proto; - u32 i; + struct bpf_call_arg_meta meta; + struct bpf_func_proto *fn; int ret, err; + u32 i; + + memset(&meta, 0, sizeof(meta)); + meta.btf = btf; + meta.subprog_may_change_pkt = sub->changes_pkt_data; + meta.func_name = bpf_subprog_name(env, subprog); ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), NULL, NULL); if (err) @@ -9706,125 +11344,33 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); - args = btf_params(func_proto); - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, - bpf_subprog_name(env, subprog), btf, args); - if (ret) - return ret; - /* check that BTF function arguments match actual types that the - * verifier sees. + fn = &env->bpf_subprog_scratch; + gen_subprog_arg_proto(sub, btf, func_proto, fn); + meta.fn = fn; + meta.func_proto = func_proto; + + /* + * Check if all register arguments declared in BTF are initialized. + * If they aren't the BTF is unreliable. */ - for (i = 0; i < sub->arg_cnt; i++) { - argno_t argno = argno_from_arg(i + 1); - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - struct bpf_subprog_arg_info *arg = &sub->args[i]; - - if (arg->arg_type == ARG_ANYTHING) { - if (reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); + if (!bpf_subprog_is_global(env, subprog)) { + for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) + if (caller->regs[BPF_REG_1 + i].type == NOT_INIT) return -EINVAL; - } - } else if (arg->arg_type & PTR_UNTRUSTED) { - /* - * Anything is allowed for untrusted arguments, as these are - * read-only and probe read instructions would protect against - * invalid memory access. - */ - } else if (arg->arg_type == ARG_PTR_TO_CTX) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); - if (ret < 0) - return ret; - /* If function expects ctx type in BTF check that caller - * is passing PTR_TO_CTX. - */ - if (reg->type != PTR_TO_CTX) { - bpf_log(log, "%s expects pointer to ctx\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); - if (ret < 0) - return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, - NULL)) - return -EINVAL; - /* - * PTR_TO_PACKET get passed as PTR_TO_MEM, preventing - * us from adjusting bounds tracking info. - */ - if ((reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) && - sub->changes_pkt_data) { - bpf_log(log, "%s is a packet pointer, but func#%d may change packet data\n", - reg_arg_name(env, argno), subprog); - return -EINVAL; - } - if (!(arg->arg_type & PTR_MAYBE_NULL) && - (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { - bpf_log(log, "%s is expected to be non-NULL\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { - /* - * Can pass any value and the kernel won't crash, but - * only PTR_TO_ARENA or SCALAR make sense. Everything - * else is a bug in the bpf program. Point it out to - * the user at the verification time instead of - * run-time debug nightmare. - */ - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a pointer to arena or scalar.\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); - if (ret) - return ret; - - ret = process_dynptr_func(env, reg, argno, env->insn_idx, - bpf_subprog_name(env, subprog), arg->arg_type, - &ref_obj, NULL); - if (ret) - return ret; - } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { - struct bpf_call_arg_meta meta; - int err; - - if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { - err = mark_arg_precision(env, argno); - if (err) - return err; - continue; - } - - memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, - bpf_subprog_name(env, subprog)); - err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); - if (err) - return err; - } else { - verifier_bug(env, "unrecognized %s type %d", - reg_arg_name(env, argno), arg->arg_type); - return -EFAULT; - } } - - return 0; + return check_func_args(env, &meta, env->insn_idx); } -/* Compare BTF of a function call with given bpf_reg_state. +/* + * Check that call-site argument states match a subprog's BTF signature. + * * Returns: * EFAULT - there is a verifier bug. Abort verification. - * EINVAL - there is a type mismatch or BTF is not available. + * Other errors - there is a type mismatch or BTF is not available. * 0 - BTF matches with what bpf_reg_state expects. - * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. */ -static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs) +static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog) { struct bpf_prog *prog = env->prog; struct btf *btf = prog->aux->btf; @@ -9841,7 +11387,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, if (prog->aux->func_info_aux[subprog].unreliable) return -EINVAL; - err = btf_check_func_arg_match(env, subprog, btf, regs); + err = btf_check_func_arg_match(env, subprog, btf); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. * In such cases mark the function as unreliable from BTF point of view. @@ -9860,7 +11406,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins int err; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; @@ -9891,8 +11437,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins is_async_cb_sleepable(env, insn)); if (IS_ERR(async_cb)) return PTR_ERR(async_cb); + if (is_async_cb_sleepable(env, insn)) + mark_sleepable_async_cb(env); callee = async_cb->frame[0]; - callee->async_entry_cnt = caller->async_entry_cnt + 1; + callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; /* Convert bpf_timer_set_callback() args into timer callback args */ err = set_callee_state_cb(env, caller, callee, insn_idx); @@ -9902,6 +11450,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins return 0; } + /* + * The address of the callback might be taken in another function or + * read from a map, see check_func_ptr_read(). Like with callx it's + * known here what is called. + */ + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -9923,89 +11480,21 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit); -static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, - int *insn_idx) +/* + * Call of a static subprog. The callee is verified in the context of + * the caller, hence set up a new frame and continue from the first + * instruction of the callee. + */ +static int check_static_func_call(struct bpf_verifier_env *env, int subprog, + int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; struct bpf_subprog_info *caller_info; u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; - int err, subprog, target_insn; - - target_insn = *insn_idx + insn->imm + 1; - subprog = bpf_find_subprog(env, target_insn); - if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", - target_insn)) - return -EFAULT; + int err; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); - if (err == -EFAULT) - return err; - if (bpf_subprog_is_global(env, subprog)) { - struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); - const char *sub_name = bpf_subprog_name(env, subprog); - const char *operation; - bool returns_void; - - if (env->cur_state->active_locks) { - verbose(env, "global function calls are not allowed while holding a lock,\n" - "use static function instead\n"); - operation = bpf_diag_fmt(env, "global function %s()", sub_name); - bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, - "Release the lock before calling the global function, or use a static function instead."); - return -EINVAL; - } - - if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { - verbose(env, "sleepable global function %s() called in %s\n", - sub_name, non_sleepable_context_description(env)); - operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); - bpf_diag_ctx_forbidden(env, *insn_idx, operation, - "Move the call outside the critical section, or use a non-sleepable function."); - return -EINVAL; - } - - if (err) { - verbose(env, "Caller passes invalid args into func#%d ('%s')\n", - subprog, sub_name); - return err; - } - - if (env->log.level & BPF_LOG_LEVEL) - verbose(env, "Func#%d ('%s') is global and assumed valid.\n", - subprog, sub_name); - sub_aux->called[in_sleepable_context(env)] = true; - returns_void = subprog_returns_void(env, subprog); - if (env->subprog_info[subprog].changes_pkt_data) - clear_all_pkt_pointers(env); - if (returns_void) - bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); - else - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); - clear_caller_saved_regs(env, caller->regs); - invalidate_outgoing_stack_args(env, cur_func(env)); - - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ - if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); - bpf_diag_mod_end(env); - } - - if (env->subprog_info[subprog].might_throw) { - struct bpf_verifier_state *branch; - - branch = push_stack(env, *insn_idx + 1, *insn_idx, false); - if (IS_ERR(branch)) { - verbose(env, "failed to push state for global subprog exception path\n"); - return PTR_ERR(branch); - } - return process_bpf_exit_full(env, NULL, true); - } - - /* continue with next insn after call */ - return 0; - } /* * Track caller's total stack arg count (incoming + max outgoing). @@ -10017,7 +11506,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (stack_arg_cnt > caller_info->stack_arg_cnt) caller_info->stack_arg_cnt = stack_arg_cnt; - /* for regular function entry setup new frame and continue + /* + * For regular function entry setup new frame and continue * from that frame. */ err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); @@ -10040,6 +11530,194 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_verifier_state *state = env->cur_state; + struct bpf_func_state *caller; + int err, subprog, target_insn; + u32 i, nregs; + + target_insn = *insn_idx + insn->imm + 1; + subprog = bpf_find_subprog(env, target_insn); + if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", + target_insn)) + return -EFAULT; + + caller = state->frame[state->curframe]; + err = btf_check_subprog_call(env, subprog); + if (err == -EFAULT) + return err; + if (bpf_subprog_is_global(env, subprog)) { + struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); + const char *operation; + bool returns_void; + + if (env->cur_state->active_locks) { + verbose(env, "global function calls are not allowed while holding a lock,\n" + "use static function instead\n"); + operation = bpf_diag_fmt(env, "global function %s()", sub_name); + bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, + "Release the lock before calling the global function, or use a static function instead."); + return -EINVAL; + } + + if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + + verbose(env, "sleepable global function %s() called in %s\n", + sub_name, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the call outside the critical section, or use a non-sleepable function."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function."; + operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion); + return -EINVAL; + } + + if (err) { + verbose(env, "Caller passes invalid args into func#%d ('%s')\n", + subprog, sub_name); + return err; + } + + if (env->log.level & BPF_LOG_LEVEL) + verbose(env, "Func#%d ('%s') is global and assumed valid.\n", + subprog, sub_name); + if (in_sleepable_context(env)) + sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] = true; + else if (has_rcu_read_lock(env)) + sub_aux->called[BPF_FUNC_CTX_NON_SLEEPABLE] = true; + else + sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK] = true; + returns_void = subprog_returns_void(env, subprog); + if (env->subprog_info[subprog].changes_pkt_data) + clear_all_pkt_pointers(env); + if (returns_void) + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + else + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); + clear_caller_saved_regs(env, caller->regs); + invalidate_outgoing_stack_args(env, cur_func(env)); + /* + * What the callers of a global function hold is not known + * while it is verified. + */ + if (!env->subprog_info[subprog].changes_pkt_data && + (state->frame[0]->subprogno || holds_pkt_pointers(env))) + sub_aux->called_with_pkt_ptrs = true; + + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ + if (!returns_void) { + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + mark_reg_unknown(env, caller->regs, ret_regs[0]); + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + } + + if (env->subprog_info[subprog].can_unwind) { + struct bpf_verifier_state *branch; + + if (!bpf_is_unwind_goto(insn + 1)) { + verbose(env, "no unwind_goto after the call of %s()\n", sub_name); + return -EINVAL; + } + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + branch->unwound = true; + } + + if (env->subprog_info[subprog].might_throw) { + struct bpf_verifier_state *branch; + + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); + if (IS_ERR(branch)) { + verbose(env, "failed to push state for global subprog exception path\n"); + return PTR_ERR(branch); + } + return process_bpf_exit_full(env, NULL, true); + } + + /* continue with next insn after call */ + return 0; + } + + return check_static_func_call(env, subprog, insn_idx); +} + +/* + * callx dst_reg: call a bpf subprog whose address is in dst_reg. + * + * The address of a subprog is either loaded into a register by ld_imm64 with + * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see + * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all + * possible callees of callx are discovered by add_subprogs() and are reachable + * in the control flow graph before the main verification pass begins. + * PTR_TO_FUNC register identifies the callee, so from here on callx is verified + * as a direct call of that static subprog. + */ +static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *reg; + const char *reason; + int err, subprog; + + err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + + reg = reg_state(env, insn->dst_reg); + if (reg->type != PTR_TO_FUNC) { + verbose(env, "R%d has type %s, expected func\n", insn->dst_reg, + reg_type_str(env, reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s, but callx can only call through the address of a static BPF function.", + insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type)); + bpf_diag_register_type( + env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason, + "Load the address of a static BPF function into the register before callx."); + return -EACCES; + } + + /* + * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address + * of a subprog can be called. + */ + err = check_ptr_off_reg(env, reg, insn->dst_reg); + if (err) + return err; + + /* the callx is valid, but only a JIT can execute it */ + err = require_callx_jit(env); + if (err) + return err; + + /* PTR_TO_FUNC is a pointer to a static subprog */ + subprog = reg->subprogno; + err = btf_check_subprog_call(env, subprog); + if (err == -EFAULT) + return err; + + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + + return check_static_func_call(env, subprog, insn_idx); +} + int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee) @@ -10105,6 +11783,19 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, if (err) return err; + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_RHASH: + /* Hash tables run the callback under rcu_read_lock(). */ + callee->in_rcu_callback_fn = true; + break; + default: + break; + } + callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; @@ -10191,6 +11882,8 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + /* bpf_find_vma() runs the callback under mmap_lock and RCU. */ + callee->in_rcu_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -10286,6 +11979,40 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, return 0; } +static int set_rcu_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr; + u32 map_uid = caller->regs[BPF_REG_2].map_uid; + + /* + * callback_fn(struct bpf_map *map, void *key, void *value); + */ + callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; + __mark_reg_known_zero(&callee->regs[BPF_REG_1]); + callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; + + callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; + + callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; + __mark_reg_known_zero(&callee->regs[BPF_REG_3]); + callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; + + /* unused */ + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_async_callback_fn = true; + callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); + return 0; +} + static bool is_rbtree_lock_required_kfunc(u32 btf_id); static void account_processed_insn(struct bpf_verifier_env *env) @@ -10364,11 +12091,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10380,6 +12111,19 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } caller = state->frame[state->curframe - 1]; + if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx])) { + /* the helper or the kfunc that called it doesn't expect that */ + if (callee->in_callback_fn) { + verbose(env, "callback cannot unwind\n"); + return -EINVAL; + } + if (!bpf_is_unwind_goto(&env->prog->insnsi[callee->callsite + 1])) { + verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n", + callee->callsite); + return -EINVAL; + } + state->unwound = true; + } if (callee->in_callback_fn) { if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); @@ -10403,12 +12147,26 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; - bpf_diag_mod_end(env); + /* + * return to the caller whatever the callee had in the + * return register(s) + */ + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } + /* Invalidate callee-local dynptrs and their slices before the frame goes away. */ + err = destroy_dynptrs_in_stack_slots(env, callee, 0, + callee->allocated_stack / BPF_REG_SIZE - 1); + if (err) + return err; + /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, * there function call logic would reschedule callback visit. If iteration * converges is_state_visited() would prune that visit eventually. @@ -10796,22 +12554,37 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, return *ptr && (*ptr)->func ? 0 : -EINVAL; } +/* Is any frame on the current path a callback that runs under RCU? */ +static bool in_rcu_callback(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int i; + + for (i = 0; i <= state->curframe; i++) + if (state->frame[i]->in_rcu_callback_fn) + return true; + return false; +} + /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !in_rcu_cs(env); + return !in_rcu_cs(env) && !in_rcu_callback(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) { - if (env->cur_state->active_rcu_locks) - return "rcu_read_lock region"; - if (env->cur_state->active_preempt_locks) - return "non-preemptible region"; - if (env->cur_state->active_irq_id) - return "IRQ-disabled region"; - if (env->cur_state->active_locks) - return "lock region"; + if (in_sleepable(env)) { + if (env->cur_state->active_rcu_locks) + return "rcu_read_lock region"; + if (env->cur_state->active_preempt_locks) + return "non-preemptible region"; + if (env->cur_state->active_irq_id) + return "IRQ-disabled region"; + if (env->cur_state->active_locks) + return "lock region"; + return "callback running under rcu_read_lock"; + } return "non-sleepable prog"; } @@ -10896,19 +12669,24 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn memset(&meta, 0, sizeof(meta)); - err = check_func_proto(fn, &meta); + err = check_func_proto(env, fn, &meta); if (err) { verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); return err; } if (fn->might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper."; operation = bpf_diag_fmt(env, "sleepable helper %s#%d", func_id_name(func_id), func_id); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the helper call outside the critical section, or use a non-sleepable helper."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EINVAL; } @@ -10917,13 +12695,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.func_name = func_id_name(func_id); meta.fn = fn; - /* check args */ - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, insn_idx); - if (err) - return err; - } + err = check_func_args(env, &meta, insn_idx); + if (err) + return err; err = record_func_map(env, &meta, func_id, insn_idx); if (err) @@ -10935,16 +12711,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs = cur_regs(env); - /* Mark slots with STACK_MISC in case of raw mode, stack offset - * is inferred from register state. - */ - for (i = 0; i < meta.arg_raw_mem.size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, - argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, - BPF_WRITE, -1, false, false); - if (err) - return err; - } + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; @@ -11065,8 +12834,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) return -EFAULT; - if (dynptr_type == BPF_DYNPTR_TYPE_SKB || - dynptr_type == BPF_DYNPTR_TYPE_SKB_META) + if (dynptr_write_changes_pkt_data(dynptr_type)) /* this will trigger clear_all_pkt_pointers(), which will * invalidate all dynptr slices associated with the skb */ @@ -11364,6 +13132,37 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. + */ +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -11474,6 +13273,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__ctx_out"); +} + static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__arena__nullable") || @@ -11509,7 +13313,8 @@ enum { KF_ARG_RES_SPIN_LOCK_ID, KF_ARG_TASK_WORK_ID, KF_ARG_PROG_AUX_ID, - KF_ARG_TIMER_ID + KF_ARG_TIMER_ID, + KF_ARG_RCU_HEAD_ID }; BTF_ID_LIST(kf_arg_btf_ids) @@ -11523,6 +13328,7 @@ BTF_ID(struct, bpf_res_spin_lock) BTF_ID(struct, bpf_task_work) BTF_ID(struct, bpf_prog_aux) BTF_ID(struct, bpf_timer) +BTF_ID(struct, bpf_rcu_head) static bool __is_kfunc_ptr_arg_type(const struct btf *btf, const struct btf_param *arg, int type) @@ -11581,6 +13387,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID); } +static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg) +{ + return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID); +} + static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg) { return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); @@ -11635,13 +13446,39 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11650,58 +13487,128 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; + } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; - } - if (!btf_type_is_scalar(member_type)) - return false; + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; } -enum kfunc_ptr_arg_type { - KF_ARG_CONST_MEM_SIZE, - KF_ARG_MEM_SIZE, - KF_ARG_CONST, - KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, - KF_ARG_ANYTHING, - KF_ARG_PTR_TO_CTX, - KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ - KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ - KF_ARG_PTR_TO_DYNPTR, - KF_ARG_PTR_TO_ITER, - KF_ARG_PTR_TO_LIST_HEAD, - KF_ARG_PTR_TO_LIST_NODE, - KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ - KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_CALLBACK, - KF_ARG_PTR_TO_RB_ROOT, - KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_CONST_STR, - KF_ARG_CONST_MAP_PTR, - KF_ARG_PTR_TO_TIMER, - KF_ARG_PTR_TO_WORKQUEUE, - KF_ARG_PTR_TO_IRQ_FLAG, - KF_ARG_PTR_TO_RES_SPIN_LOCK, - KF_ARG_PTR_TO_TASK_WORK, - KF_ARG_PTR_TO_ARENA, -}; +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); +} + +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type, u32 *arg_size) +{ + const struct btf_param *args; + const struct btf_type *ref_t, *resolve_ret; + const struct btf *btf; + const char *ref_tname; + u32 ref_id; + + if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE) + return resolve_map_arg_type(env, meta, arg_type); + + if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) + return 0; + + if (!is_kfunc(meta) || arg_type_is_release(*arg_type) || + base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) + return 0; + + args = btf_params(meta->func_proto); + ref_id = *meta->fn->arg_btf_id[arg]; + btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf; + ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + + if (!btf_type_is_scalar_struct(env, btf, ref_t)) + return 0; + + resolve_ret = btf_resolve_size(btf, ref_t, arg_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, + PTR_ERR(resolve_ret)); + return -EINVAL; + } + *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE | + (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT)); + + return 0; +} + +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } +} enum special_kfunc_type { KF_bpf_obj_new_impl, @@ -11768,6 +13675,8 @@ enum special_kfunc_type { KF___bpf_trap, KF_bpf_task_work_schedule_signal, KF_bpf_task_work_schedule_resume, + KF_bpf_call_rcu, + KF_bpf_call_rcu_tasks_trace, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, KF_bpf_session_is_return, @@ -11858,14 +13767,36 @@ BTF_ID(func, bpf_dynptr_file_discard) BTF_ID(func, __bpf_trap) BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) +BTF_ID(func, bpf_call_rcu) +BTF_ID(func, bpf_call_rcu_tasks_trace) +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) +#else +BTF_ID_UNUSED +BTF_ID_UNUSED +#endif #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]); +} + +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]); +} + +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]); +} + static bool is_bpf_obj_new_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_obj_new] || @@ -11911,6 +13842,12 @@ static bool is_bpf_rbtree_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; } +static bool is_call_rcu_kfunc(u32 func_id) +{ + return func_id == special_kfunc_list[KF_bpf_call_rcu] || + func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace]; +} + static bool is_task_work_add_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || @@ -11927,52 +13864,159 @@ static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]); } static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]); } static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]); } static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]); } +/* These write to what their first argument, a dynptr, points to */ +BTF_SET_START(dynptr_write_kfuncs) +BTF_ID(func, bpf_dynptr_copy) +BTF_ID(func, bpf_dynptr_memset) +#ifdef CONFIG_BPF_EVENTS +BTF_ID(func, bpf_probe_read_user_dynptr) +BTF_ID(func, bpf_probe_read_kernel_dynptr) +BTF_ID(func, bpf_probe_read_user_str_dynptr) +BTF_ID(func, bpf_probe_read_kernel_str_dynptr) +BTF_ID(func, bpf_copy_from_user_dynptr) +BTF_ID(func, bpf_copy_from_user_str_dynptr) +BTF_ID(func, bpf_copy_from_user_task_dynptr) +BTF_ID(func, bpf_copy_from_user_task_str_dynptr) +#endif +BTF_SET_END(dynptr_write_kfuncs) + +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta) +{ + return meta->btf == btf_vmlinux && + btf_id_set_contains(&dynptr_write_kfuncs, meta->func_id); +} + +/* + * A write through the dynptr of an skb may reallocate its head. Before the + * arguments are checked nobody knows what kind of dynptr it is. + */ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]) || + is_kfunc_dynptr_write(meta); +} + +static u32 kfunc_abi_slots(const struct btf_func_model *fm) +{ + const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi(); + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 i, nslots, slots = 0; + + for (i = 0; i < fm->nr_args; i++) + slots += btf_func_model_arg_slots(fm, i); + + if (!abi) + return slots; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + for (i = 0; i < nslots; i++) + if (pos_of_slot[i] + 1 > slots) + slots = pos_of_slot[i] + 1; + + return slots; +} + +static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec) +{ + const struct btf_member *member; + const struct btf_type *mt; + u32 align, i; + + while (btf_type_is_array(t)) + t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL); + + if (btf_type_is_int(t)) + return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE; + if (!btf_type_is_struct(t)) + return BPF_REG_SIZE; + if (rec >= BTF_MEMBER_MAX_DEPTH) + return 0; + + for_each_member(i, t, member) { + mt = btf_type_skip_modifiers(btf, member->type, NULL); + align = __btf_func_arg_align(btf, mt, rec + 1); + if (!align) + return 0; + if (align > BPF_REG_SIZE) + return 2 * BPF_REG_SIZE; + } + return BPF_REG_SIZE; +} + +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t) +{ + return __btf_func_arg_align(btf, t, 0); } static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_param *args, int arg, int nargs) + const struct btf_param *args, int arg, int nargs, u32 slot, + struct bpf_func_proto *proto) { - const struct btf_type *t, *ref_t = NULL; - argno_t argno = argno_from_arg(arg + 1); + const struct btf_type *t, *ref_t = NULL, *resolve_ret; + const u32 *ref_id_ptr = NULL; + argno_t argno = argno_from_arg(slot + 1); const char *ref_tname = NULL; + u32 ref_id, type_size; int arg_type; + proto->arg_btf_id[arg] = NULL; + + if (is_kfunc_arg_prog_aux(meta->btf, &args[arg])) + return ARG_PTR_TO_PROG_AUX; + + if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg)) + return ARG_IGNORE; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); /* Scalar arguments are classified from their BTF suffix/name alone. */ if (btf_type_is_scalar(t)) { if (is_kfunc_arg_constant(meta->btf, &args[arg])) - return KF_ARG_CONST; + return ARG_CONST_SCALAR; if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) - return KF_ARG_CONST_MEM_SIZE; + return ARG_CONST_MEM_SIZE; if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) - return KF_ARG_MEM_SIZE; + return ARG_MEM_SIZE; if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) - return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; - return KF_ARG_ANYTHING; + return ARG_CONST_ALLOC_SIZE_OR_ZERO; + return ARG_SCALAR; + } + + if (btf_type_is_struct(t)) { + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + verbose(env, + "%s type %s has size %u, only 1 to %d bytes " + "can be passed by value\n", + reg_arg_name(env, argno), btf_type_str(t), t->size, + 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_type_is_scalar_struct(env, meta->btf, t)) { + verbose(env, "%s type %s is not composed of scalars\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + return ARG_SCALAR; } if (!btf_type_is_ptr(t)) { @@ -11980,54 +14024,87 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; } - ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + /* Keep a pointer to the BTF field containing the resolved referent ID. */ + ref_id_ptr = &t->type; + ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id); + while (*ref_id_ptr != ref_id) + ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type; ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of - * arguments, we resolve it to a known kfunc_ptr_arg_type. + * arguments, we resolve it to a known bpf_arg_type. */ - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - arg_type = KF_ARG_PTR_TO_CTX; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie])) + arg_type = ARG_PTR_TO_CTX; else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - arg_type = KF_ARG_PTR_TO_CTX; + arg_type = ARG_PTR_TO_CTX; else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + arg_type = ARG_PTR_TO_ALLOC_BTF_ID; else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; - else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_DYNPTR; - else if (is_kfunc_arg_iter(meta, arg, &args[arg])) - arg_type = KF_ARG_PTR_TO_ITER; + arg_type = ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) { + arg_type = ARG_PTR_TO_DYNPTR; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) + arg_type |= DYNPTR_TYPE_SKB; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) + arg_type |= DYNPTR_TYPE_XDP; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) + arg_type |= DYNPTR_TYPE_SKB_META; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) + /* OBJ_RELEASE for the latter comes from KF_RELEASE below */ + arg_type |= DYNPTR_TYPE_FILE; + } else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = ARG_PTR_TO_ITER; else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_HEAD; + arg_type = ARG_PTR_TO_LIST_HEAD; else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_NODE; + arg_type = ARG_PTR_TO_LIST_NODE; else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_ROOT; + arg_type = ARG_PTR_TO_RB_ROOT; else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_NODE; + arg_type = ARG_PTR_TO_RB_NODE; else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CONST_STR; + arg_type = ARG_PTR_TO_CONST_STR; else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) - arg_type = KF_ARG_CONST_MAP_PTR; + arg_type = ARG_CONST_MAP_PTR; else if (is_kfunc_arg_map(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_BTF_ID; + arg_type = ARG_PTR_TO_BTF_ID; else if (is_kfunc_arg_wq(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_WORKQUEUE; + arg_type = ARG_PTR_TO_WORKQUEUE; else if (is_kfunc_arg_timer(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TIMER; + arg_type = ARG_PTR_TO_TIMER; else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TASK_WORK; + arg_type = ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_RCU_HEAD; else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + arg_type = ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; - else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CALLBACK; + arg_type = ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) { + if (!btf_type_is_scalar(ref_t)) { + verbose(env, "%s __ctx_out argument must point to a scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE; + } else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_FUNC; else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { if (!bpf_jit_supports_arena_args()) { verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", @@ -12050,20 +14127,20 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * whether the JIT rebases it to the arena base or preserves NULL. * The common nullable path below records that verifier property. */ - arg_type = KF_ARG_PTR_TO_ARENA; + arg_type = ARG_PTR_TO_ARENA; } else if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM; + arg_type = ARG_PTR_TO_MEM | MEM_WRITE; } else if (btf_type_is_struct(ref_t)) - /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ - arg_type = KF_ARG_PTR_TO_BTF_ID; + /* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */ + arg_type = ARG_PTR_TO_BTF_ID; else { /* * Otherwise this is a fixed-size memory buffer supported by @@ -12071,26 +14148,64 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE; } + if (is_kfunc_arg_uninit(meta->btf, &args[arg])) + arg_type |= MEM_UNINIT; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) arg_type |= PTR_MAYBE_NULL; + /* + * Only the first argument of a KF_RELEASE kfunc releases anything, and + * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it. + */ + if (is_kfunc_release(meta) && arg == 0) + arg_type |= OBJ_RELEASE; + + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + if (is_kfunc_arg_map(meta->btf, &args[arg])) + proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP]; + else + proto->arg_btf_id[arg] = ref_id_ptr; + + arg_type |= PTR_TRUSTED; + + /* + * A KF_RCU kfunc accepts an RCU-protected pointer where it would + * otherwise demand a referenced or trusted one. Other argument kinds + * have their own provenance requirements and must not inherit MEM_RCU. + */ + if (is_kfunc_rcu(meta)) + arg_type |= MEM_RCU; + } + return arg_type; } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto) + const struct btf_func_model *fm, struct bpf_func_proto *proto) { + const struct bpf_jit_arg_abi *abi; const struct btf *btf = meta->btf; const struct btf_param *args; - u32 i, nargs; + const struct btf_type *t; + u32 i, nargs, slots_used; int arg_type; args = (const struct btf_param *)(meta->func_proto + 1); @@ -12100,54 +14215,84 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - meta->func_name, nargs); - return -ENOTSUPP; + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + if (!btf_func_arg_align(btf, t)) { + verbose(env, + "Function %s arg#%d type %s nests structs more than " + "%d levels deep\n", + meta->func_name, i, btf_type_str(t), + BTF_MEMBER_MAX_DEPTH); + return -EINVAL; + } + } + slots_used += nslots; } - for (i = 0; i < nargs; i++) { - if (is_kfunc_arg_prog_aux(btf, &args[i]) || - is_kfunc_arg_ignore(btf, &args[i]) || - is_kfunc_arg_implicit(meta, i)) - continue; + if (slots_used > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name, + slots_used, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + abi = bpf_jit_arg_abi(); + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (!abi && nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + verbose(env, + "Function %s arg#%d type %s cannot be passed at " + "argument slot %d on this architecture\n", + meta->func_name, i, btf_type_str(t), slots_used); + return -EINVAL; + } + slots_used += nslots; + + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, + slots_used - nslots, proto); if (arg_type < 0) return arg_type; proto->arg_type[i] = arg_type; } - return 0; + if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d argument slots\n", + meta->func_name, slots_used); + return -ENOTSUPP; + } + + return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } -static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, - const struct btf_type *ref_t, - const char *ref_tname, u32 ref_id, - struct bpf_call_arg_meta *meta, - int arg, argno_t argno) +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx) { - const struct btf_type *reg_ref_t; - bool strict_type_match = false; + bool taking_projection, struct_same, strict_type_match = false; + const struct btf_type *arg_t, *reg_t; + const char *arg_tname, *reg_tname; const struct btf *reg_btf; - const char *reg_ref_tname; - bool taking_projection; - bool struct_same; - u32 reg_ref_id; + u32 reg_btf_id; if (base_type(reg->type) == PTR_TO_BTF_ID) { reg_btf = reg->btf; - reg_ref_id = reg->btf_id; + reg_btf_id = reg->btf_id; } else { reg_btf = btf_vmlinux; - reg_ref_id = *reg2btf_ids[base_type(reg->type)]; + reg_btf_id = *reg2btf_ids[base_type(reg->type)]; } - /* Enforce strict type matching for calls to kfuncs that are acquiring - * or releasing a reference, or are no-cast aliases. We do _not_ - * enforce strict matching for kfuncs by default, + /* + * Enforce strict type matching for arguments that release a reference, + * or are no-cast aliases. We do _not_ enforce strict matching by default, * as we want to enable BPF programs to pass types that are bitwise * equivalent without forcing them to explicitly cast with something * like bpf_cast_to_kern_ctx(). @@ -12169,27 +14314,30 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, * btf_struct_ids_match() to walk the struct at the 0th offset, and * resolve types. */ - if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) || - btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) + if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) || + (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, + arg_btf, arg_btf_id))) strict_type_match = true; - WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off)); + arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id); + arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off); + reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, ®_btf_id); + reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off); - reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); - reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); - struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match, - !type_is_alloc(reg->type)); + struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id, + reg->var_off.value, arg_btf, arg_btf_id, + strict_type_match, !type_is_alloc(reg->type)); + /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. */ - taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); + taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname); if (!taking_projection && !struct_same) { - verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n", + verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno), - btf_type_str(reg_ref_t), reg_ref_tname); + btf_type_str(arg_t), arg_tname, + reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname); return -EINVAL; } return 0; @@ -12201,15 +14349,15 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; - if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { irq_save = true; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) kfunc_class = IRQ_LOCK_KFUNC; - } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) { irq_save = false; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) kfunc_class = IRQ_LOCK_KFUNC; } else { verifier_bug(env, "unknown irq flags kfunc"); @@ -12402,12 +14550,20 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta) { - return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; + int flags = PROCESS_RES_LOCK; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) + flags |= PROCESS_SPIN_LOCK; + else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) && + !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + return 0; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + flags |= PROCESS_LOCK_IRQ; + return flags; } static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) @@ -12422,6 +14578,55 @@ static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 of return *kfunc.flags & KF_SPINLOCK_SAFE; } +/* + * True if insn calls a helper/kfunc that requires one of its arguments to + * be a stack pointer with a constant offset. + */ +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + const struct bpf_func_proto *fn; + struct bpf_kfunc_desc *desc; + u32 *flags, btf_id; + int i; + + if (bpf_helper_call(insn)) { + if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) + return false; + } else if (bpf_pseudo_kfunc_call(insn)) { + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) + return false; + fn = &desc->proto; + } else { + return false; + } + + /* Both initialized and uninitialized stack dynptrs need a fixed offset. */ + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) + if (arg_type_is_dynptr(fn->arg_type[i])) + return true; + + /* vmlinux kfuncs only */ + if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0) + return false; + btf_id = insn->imm; + + flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog); + if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY))) + return true; + + if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] || + btf_id == special_kfunc_list[KF_bpf_local_irq_save] || + btf_id == special_kfunc_list[KF_bpf_local_irq_restore]) + return true; + + return false; +} + static bool is_sync_callback_calling_kfunc(u32 btf_id) { return is_bpf_rbtree_add_kfunc(btf_id); @@ -12430,7 +14635,8 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id) static bool is_async_callback_calling_kfunc(u32 btf_id) { return is_bpf_wq_set_callback_kfunc(btf_id) || - is_task_work_add_kfunc(btf_id); + is_task_work_add_kfunc(btf_id) || + is_call_rcu_kfunc(btf_id); } bool bpf_is_throw_kfunc(struct bpf_insn *insn) @@ -12684,707 +14890,6 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - int insn_idx) -{ - const char *func_name = meta->func_name, *ref_tname; - struct bpf_func_state *caller = cur_func(env); - struct bpf_reg_state *regs = cur_regs(env); - const struct btf *btf = meta->btf; - const struct btf_param *args; - struct btf_record *rec; - u32 i, nargs; - int ret; - - args = (const struct btf_param *)(meta->func_proto + 1); - nargs = btf_type_vlen(meta->func_proto); - - ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); - if (ret) - return ret; - - /* Check that BTF function arguments match actual types that the - * verifier sees. - */ - for (i = 0; i < nargs; i++) { - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - const struct btf_type *t, *ref_t, *resolve_ret; - enum bpf_arg_type arg_type = ARG_DONTCARE; - argno_t argno = argno_from_arg(i + 1); - int regno = reg_from_argno(argno); - bool btf_id_fixed_off_ok = true; - u32 ref_id = args[i].type, type_size; - int kf_arg_type = meta->fn->arg_type[i]; - - if (is_kfunc_arg_prog_aux(btf, &args[i])) { - /* Reject repeated use bpf_prog_aux */ - if (meta->arg_prog) { - verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); - return -EFAULT; - } - if (regno < 0) { - verbose(env, "%s prog->aux cannot be a stack argument\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - meta->arg_prog = true; - cur_aux(env)->arg_prog = regno; - continue; - } - - if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i)) - continue; - - t = btf_type_skip_modifiers(btf, args[i].type, NULL); - - if (btf_type_is_ptr(t)) { - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - if (btf_type_is_ptr(t) && - (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !type_may_be_null(kf_arg_type)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); - verbose(env, "Possibly NULL pointer passed to trusted %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Add a NULL check and call the kfunc only on the non-NULL path.", - "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", - expected_type); - return -EACCES; - } - - if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_name, reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", - "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", - expected_type); - return -EINVAL; - } - - if (reg_is_referenced(env, reg)) - update_ref_obj(&meta->ref_obj, reg); - - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) { - ret = mark_arg_precision(env, argno); - if (ret) - return ret; - continue; - } - - if (is_kfunc_arg_map(btf, &args[i])) { - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - case KF_ARG_CONST_MEM_SIZE: - case KF_ARG_MEM_SIZE: - case KF_ARG_ANYTHING: - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - case KF_ARG_PTR_TO_BTF_ID: - case KF_ARG_CONST_MAP_PTR: - case KF_ARG_PTR_TO_ITER: - case KF_ARG_PTR_TO_LIST_HEAD: - case KF_ARG_PTR_TO_LIST_NODE: - case KF_ARG_PTR_TO_RB_ROOT: - case KF_ARG_PTR_TO_RB_NODE: - case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_CONST_STR: - case KF_ARG_PTR_TO_WORKQUEUE: - case KF_ARG_PTR_TO_TIMER: - case KF_ARG_PTR_TO_TASK_WORK: - case KF_ARG_PTR_TO_IRQ_FLAG: - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - case KF_ARG_PTR_TO_ARENA: - break; - case KF_ARG_PTR_TO_DYNPTR: - arg_type = ARG_PTR_TO_DYNPTR; - break; - case KF_ARG_PTR_TO_CTX: - arg_type = ARG_PTR_TO_CTX; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - arg_type = ARG_PTR_TO_BTF_ID; - btf_id_fixed_off_ok = false; - break; - default: - verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); - return -EFAULT; - } - - if (regno == meta->release_regno) - arg_type |= OBJ_RELEASE; - ret = __check_func_arg_reg_off(env, reg, argno, arg_type, - btf_id_fixed_off_ok); - if (ret < 0) - return ret; - - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case KF_ARG_ANYTHING: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - break; - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) - meta->r0_rdonly = true; - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-known constant size for this kfunc buffer argument.", - "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case KF_ARG_PTR_TO_CTX: - if (reg->type != PTR_TO_CTX) { - verbose(env, "%s expected pointer to ctx, but got %s\n", - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the original program context pointer or preserve it before modifying registers.", - "the kfunc expects a context pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { - ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); - if (ret < 0) - return -EINVAL; - meta->ret_btf_id = ret; - } - break; - case KF_ARG_PTR_TO_ARENA: - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a pointer to arena or scalar\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - break; - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { - if (!is_bpf_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { - if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_percpu_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer returned by the matching BPF object allocation path.", - "the kfunc expects an allocated object pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the owned object pointer before it is released or transferred.", - "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->btf == btf_vmlinux) { - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - } - break; - case KF_ARG_PTR_TO_DYNPTR: - { - enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_uninit(btf, &args[i])) - dynptr_arg_type |= MEM_UNINIT; - - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { - dynptr_arg_type |= DYNPTR_TYPE_SKB; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) { - dynptr_arg_type |= DYNPTR_TYPE_XDP; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) { - dynptr_arg_type |= DYNPTR_TYPE_SKB_META; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && - (dynptr_arg_type & MEM_UNINIT)) { - enum bpf_dynptr_type parent_type = meta->dynptr.type; - - if (parent_type == BPF_DYNPTR_TYPE_INVALID) { - verifier_bug(env, "no dynptr type for parent of clone"); - return -EFAULT; - } - - dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); - } - - ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, - dynptr_arg_type, &meta->ref_obj, &meta->dynptr); - if (ret < 0) - return ret; - break; - } - case KF_ARG_PTR_TO_ITER: - if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) { - if (!check_css_task_iter_allowlist(env)) { - verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); - return -EINVAL; - } - } - ret = process_iter_arg(env, reg, argno, insn_idx, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_LIST_HEAD: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RB_ROOT: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_LIST_NODE: - if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && - type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { - /* Allow bpf_list_front/back return value for - * __nonown_allowed list-node arguments. - */ - goto check_ok; - } - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } -check_ok: - ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RB_NODE: - if (is_bpf_rbtree_add_kfunc(meta->func_id)) { - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - } else { - if (!type_is_non_owning_ref(reg->type) && - !reg_is_referenced(env, reg)) { - verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); - return -EINVAL; - } - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "%s not allowed in rbtree cb\n", func_name); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_CONST_MAP_PTR: - if (base_type(reg->type) != CONST_PTR_TO_MAP || - type_may_be_null(reg->type)) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_map_ptr_arg(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if (base_type(reg->type) == PTR_TO_BTF_ID || - reg2btf_ids[base_type(reg->type)]) { - if (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type)) { - if (!is_kfunc_rcu(meta)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", - "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Use this kfunc with a pointer that is valid in an RCU read lock region.", - "the kfunc requires an RCU-protected pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); - if (ret < 0) - return ret; - break; - } - - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { - enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); - const char *expected_type; - - verbose(env, "%s is %s expected %s %s", - reg_arg_name(env, argno), reg_type_str(env, reg->type), - btf_type_str(ref_t), ref_tname); - if (reg2btf_type != NOT_INIT) - verbose(env, " or %s", reg_type_str(env, reg2btf_type)); - verbose(env, "\n"); - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", - "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", - expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - /* - * If the register does not contain btf id but the argument type is a pointer to - * scalar-only struct, allow verifying it as a fixed size memory. - */ - kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; - fallthrough; - case KF_ARG_PTR_TO_MEM: - if (kf_arg_type & MEM_FIXED_SIZE) { - bool known_memory; - - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; - } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, - meta, &known_memory); - if (ret < 0) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - if (known_memory) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass memory with at least the required number of accessible bytes and suitable read and write access.", - "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", - type_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", - "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", - type_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return ret; - } - } - break; - case KF_ARG_CONST_MEM_SIZE: - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - fallthrough; - case KF_ARG_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); - struct bpf_reg_state *size_reg = reg; - argno_t buff_argno = argno_from_arg(i); - enum bpf_mem_size_failure failure; - - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar length for this memory argument.", - "the kfunc expects a scalar memory size, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (bpf_register_is_null(buff_reg)) - break; - - ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, - BPF_READ | BPF_WRITE, true, meta, &failure); - if (ret < 0) { - const char *buff_arg, *size_arg; - - buff_arg = bpf_diag_arg_name(env, buff_argno); - size_arg = bpf_diag_arg_name(env, argno); - verbose(env, "%s and ", reg_arg_name(env, buff_argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, argno)); - if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { - bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, - "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", - "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", - size_arg, buff_arg); - } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { - if (reg_smin(size_reg) < 0) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may be negative because its signed minimum is %lld", - size_arg, reg_smin(size_reg)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", - size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); - } - return ret; - } - break; - } - case KF_ARG_PTR_TO_CALLBACK: - if (reg->type != PTR_TO_FUNC) { - verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); - return -EINVAL; - } - meta->subprogno = reg->subprogno; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - if (!type_is_ptr_alloc_obj(reg->type)) { - verbose(env, "%s is neither owning or non-owning ref\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", - "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) - meta->arg_owning_ref = true; - - rec = reg_btf_record(reg); - if (!rec) { - verifier_bug(env, "Couldn't find btf_record"); - return -EFAULT; - } - - if (rec->refcount_off < 0) { - verbose(env, "%s doesn't point to a type with bpf_refcount field\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - break; - case KF_ARG_PTR_TO_CONST_STR: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a const string\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", - "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = check_arg_const_str(env, reg, argno); - if (ret) - return ret; - break; - case KF_ARG_PTR_TO_WORKQUEUE: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_TIMER: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_timer_func(env, reg, argno, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_TASK_WORK: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_IRQ_FLAG: - if (reg->type != PTR_TO_STACK) { - verbose(env, "%s doesn't point to an irq flag on stack\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", - "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = process_irq_flag(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - { - int flags = PROCESS_RES_LOCK; - - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); - return -EACCES; - } - - if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s doesn't point to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) - return -EFAULT; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) - flags |= PROCESS_SPIN_LOCK; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) - flags |= PROCESS_LOCK_IRQ; - ret = process_spin_lock(env, reg, argno, flags); - if (ret < 0) - return ret; - break; - } - } - } - - return 0; -} - int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, @@ -13417,27 +14922,32 @@ int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, } /* - * Determine how many bytes a helper accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't read anything directly - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a helper's stack access through argument @arg (0-based, R1-R5). + * must_write means the verifier destroys prior state throughout size bytes. + * In other words, must_write follows the verifier's model of the + * function's behaviour, which is safe to use for is_state_visited() pruning. */ -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; const struct bpf_func_proto *fn; + enum bpf_access_type access_type; enum bpf_arg_type at; - s64 size; + bool exact_size = true; + u64 size = U32_MAX; if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) - return S64_MIN; + return info; at = fn->arg_type[arg]; + access_type = func_arg_access_type(at); switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: @@ -13460,6 +14970,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn i = aux->const_reg_vals[map_reg]; if (i < env->used_map_cnt) { + /* Bloom-filter peek reads the value buffer. */ + if (!is_key && insn->imm == BPF_FUNC_map_peek_elem && + env->used_maps[i]->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + access_type = BPF_READ; size = is_key ? env->used_maps[i]->key_size : env->used_maps[i]->value_size; goto out; @@ -13469,7 +14983,12 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * Map pointer is not known at this call site (e.g. different * maps on merged paths). Conservatively return the largest * key_size or value_size across all maps used by the program. + * This is only an upper bound, so it cannot establish a definite + * write. Map-dependent argument types can also turn an output + * into an input, so conservatively retain a read dependency. */ + exact_size = false; + access_type |= BPF_READ; val = 0; for (i = 0; i < env->used_map_cnt; i++) { struct bpf_map *map = env->used_maps[i]; @@ -13485,7 +15004,7 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn } } if (!val) - return S64_MIN; + return info; size = val; goto out; } @@ -13499,18 +15018,12 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn int size_reg = BPF_REG_1 + arg + 1; if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + size = aux->const_reg_vals[size_reg]; goto out; } - /* - * Size arg is const on each path but differs across merged - * paths. MAX_BPF_STACK is a safe upper bound for reads. - */ - if (at & MEM_UNINIT) - return 0; - return MAX_BPF_STACK; } - return S64_MIN; + /* Preserve access directions even when the extent is unknown. */ + goto out; case ARG_PTR_TO_DYNPTR: size = BPF_DYNPTR_SIZE; break; @@ -13520,71 +15033,86 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * doesn't access stack. The callback subprog does and it's * analyzed separately. */ - return 0; + return (struct arg_access_info) {}; default: - return S64_MIN; + return info; } out: - /* - * MEM_UNINIT args are write-only: the helper initializes the - * buffer without reading it. - */ - if (at & MEM_UNINIT) - return -size; - return size; + info.size = min_t(u64, size, U32_MAX); + info.may_read = !!(access_type & BPF_READ); + info.may_write = !!(access_type & BPF_WRITE); + info.must_write = info.may_write && exact_size && info.size != U32_MAX; + /* Generic unprivileged outputs retain prior initialization state. */ + if (!env->allow_uninit_stack && arg_type_is_raw_mem(at)) { + info.may_read = true; + info.must_write = false; + } + return info; } /* - * Determine how many bytes a kfunc accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't access memory through that argument (ex: not a pointer) - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a kfunc's stack access through argument slot @arg (0-based). + * As for helpers, must_write describes destruction of prior verifier state, + * rather than a guarantee that the callee writes every byte at runtime. */ -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; - u32 nargs, type_size; - s64 size; + u32 i, slot, nargs, type_size; + u64 size; if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) - return S64_MIN; + return info; btf = meta.btf; args = btf_params(meta.func_proto); nargs = btf_type_vlen(meta.func_proto); - if (arg >= nargs) - return 0; - t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + /* + * @arg is an argument slot and a 16-byte parameter takes two of them, + * so walk the parameters to find the one that starts at this slot. A + * slot holding the upper eightbyte of such a parameter belongs to no + * pointer, and neither does a slot past the last parameter. + */ + for (i = 0, slot = 0; i < nargs && slot < arg; i++) + slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + if (i >= nargs || slot != arg) + return (struct arg_access_info) {}; + + t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (!btf_type_is_ptr(t)) - return 0; + return (struct arg_access_info) {}; /* dynptr: fixed 16-byte on-stack representation */ - if (is_kfunc_arg_dynptr(btf, &args[arg])) { + if (is_kfunc_arg_dynptr(btf, &args[i])) { size = BPF_DYNPTR_SIZE; goto out; } - /* ptr + __sz/__szk pair: size is in the next register */ - if (arg + 1 < nargs && - (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || - btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { + /* ptr + __sz/__szk pair: the size follows the pointer */ + if (i + 1 < nargs && + (btf_param_match_suffix(btf, &args[i + 1], "__sz") || + btf_param_match_suffix(btf, &args[i + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; - if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + if (size_reg <= MAX_BPF_FUNC_REG_ARGS && + (aux->const_reg_mask & BIT(size_reg))) { + size = aux->const_reg_vals[size_reg]; goto out; } - return MAX_BPF_STACK; + /* Unknown size: the read may extend anywhere up to the frame top. */ + return info; } /* fixed-size pointed-to type: resolve via BTF */ @@ -13594,14 +15122,99 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * goto out; } - return S64_MIN; + return info; out: + info.size = min_t(u64, size, U32_MAX); /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) - return -size; - if (is_kfunc_arg_uninit(btf, &args[arg])) - return -size; - return size; + info.may_read = false; + if (is_kfunc_arg_uninit(btf, &args[i]) && + (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) + info.may_read = false; + info.must_write = !info.may_read && info.size != U32_MAX; + return info; +} + +/* + * Called before dead code elimination, some subprograms may be genuinely unreachable. + * Suppress diagnostics so callers can use a conservative result on error. + * The main pass would report errors for reachable subprograms. + */ +static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) +{ + struct bpf_prog *prog = env->prog; + u32 log_level = env->log.level; + int err; + + if (!prog->aux->func_info || !prog->aux->func_info[subprog].type_id) + return false; + + env->log.level = 0; + err = btf_prepare_func_args(env, subprog); + env->log.level = log_level; + return !err; +} + +/* Prepare argument metadata for liveness; the main program is handled separately. */ +static void prepare_subprog_args(struct bpf_verifier_env *env) +{ + int i; + + for (i = 1; i < env->subprog_cnt; i++) { + if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i)) + /* Keep all argument registers live if the prototype is unavailable. */ + env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS; + } +} + +/* + * Derive stack effects for a global subprog's argument slot @arg (0-based) + * from its declared type, matching the main pass's argument checks. + */ +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) +{ + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; + struct bpf_subprog_info *sub; + int callee; + + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0) + return info; + sub = subprog_info(env, callee); + if (!try_btf_prepare_func_args(env, callee)) + return info; + + switch (base_type(sub->args[arg].arg_type)) { + case ARG_UNUSED: /* Not an argument slot. */ + case ARG_SCALAR: /* A scalar, the callee can't dereference it. */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_BTF_ID: + /* + * Trusted arguments cannot be stack pointers. + * Untrusted arguments use ARG_IGNORE: + * probe reads need no caller stack state, writes are prohibited. + */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_MEM: + /* mem_size is zero for untrusted pointers of unknown size. */ + if (sub->args[arg].mem_size) { + info.size = sub->args[arg].mem_size; + info.must_write = true; + } + return info; + case ARG_PTR_TO_DYNPTR: + /* The callee may adjust the view, but cannot reinitialize the dynptr. */ + info.size = BPF_DYNPTR_SIZE; + return info; + default: + return info; + } } /* check special kfuncs and return: @@ -13678,7 +15291,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -13715,12 +15328,12 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct btf_field *field = meta->arg_rbtree_root.field; mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); - } else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].btf_id = meta->ret_btf_id; - } else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) { ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value); if (!ret_t) { verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n", @@ -13740,8 +15353,8 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg "kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n"); return -EINVAL; } - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] || - meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) { enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type); mark_reg_known_zero(env, regs, BPF_REG_0); @@ -13756,7 +15369,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) { regs[BPF_REG_0].type |= MEM_RDONLY; } else { /* this will set env->seen_direct_write to true */ @@ -13792,7 +15405,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, proto_slots, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -13872,11 +15485,20 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } sleepable = bpf_is_kfunc_sleepable(&meta); - if (sleepable && !in_sleepable(env)) { - verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + if (sleepable && !in_sleepable_context(env)) { + const char *suggestion; + + if (in_sleepable(env)) { + verbose(env, "kernel func %s is sleepable within %s\n", + func_name, non_sleepable_context_description(env)); + suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."; + } else { + verbose(env, "program must be sleepable to call sleepable kfunc %s\n", + func_name); + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."; + } operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EACCES; } @@ -13885,10 +15507,14 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_aux->non_sleepable = true; /* Check the arguments */ - err = check_kfunc_args(env, &meta, insn_idx); + err = check_func_args(env, &meta, insn_idx); if (err < 0) return err; + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; + if ((is_bpf_obj_drop_kfunc(meta.func_id) || is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ @@ -13934,6 +15560,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + if (is_call_rcu_kfunc(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_rcu_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } + rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); @@ -13976,15 +15612,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, invalidate_rcu_protected_refs(env); } - if (sleepable && !in_sleepable_context(env)) { - verbose(env, "kernel func %s is sleepable within %s\n", - func_name, non_sleepable_context_description(env)); - operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); - return -EACCES; - } - if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); return -EACCES; @@ -14055,10 +15682,63 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. + */ + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars or arena pointers by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); + return -EINVAL; + } + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14171,14 +15851,24 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (bpf_is_kfunc_pkt_changing(&meta)) + if (is_kfunc_dynptr_write(&meta) ? + dynptr_write_changes_pkt_data(meta.dynptr_write_type) : + bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - nargs = btf_type_vlen(meta.func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + /* + * These kfuncs write to the head of the skb in place if what they write + * to is linear. Like a direct write that takes bpf_unclone_prologue(). + */ + if (is_kfunc_dynptr_write(&meta) && + dynptr_write_changes_pkt_data(meta.dynptr_write_type)) + may_access_direct_pkt_data(env, NULL, BPF_WRITE); + + proto_slots = kfunc_abi_slots(&desc->func_model); + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; - u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS; u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; if (stack_arg_cnt > caller_info->stack_arg_cnt) @@ -14189,7 +15879,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); @@ -14266,7 +15957,8 @@ enum { REASON_STACK = -5, }; -static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, +static int retrieve_ptr_limit(const struct bpf_verifier_env *env, + const struct bpf_reg_state *ptr_reg, u32 *alu_limit, bool mask_to_left) { u32 max = 0, ptr_limit = 0; @@ -14278,7 +15970,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, * offset where we would need to deal with min/max bounds is * currently prohibited for unprivileged. */ - max = MAX_BPF_STACK + mask_to_left; + max = env->stack_limit + mask_to_left; ptr_limit = -ptr_reg->var_off.value; break; case PTR_TO_MAP_VALUE: @@ -14398,7 +16090,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, (opcode == BPF_SUB && !off_is_neg); } - err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); + err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left); if (err < 0) return err; @@ -14528,7 +16220,7 @@ static int check_stack_access_for_ptr_arithmetic( return -EACCES; } - if (off >= 0 || off < -MAX_BPF_STACK) { + if (off >= 0 || off < -(int)env->stack_limit) { verbose(env, "R%d stack pointer arithmetic goes out of range, " "prohibited for !root; off=%d\n", regno, off); return -EACCES; @@ -14569,6 +16261,24 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, return 0; } +static void scalar_step_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + if (tnum_is_const(src_reg->var_off)) + scalar_step_scale(dst_reg, src_reg->var_off.value); + else + reg_step_reset(dst_reg); +} + +static void scalar_step_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + u64 amount = src_reg->var_off.value; + + if (tnum_is_const(src_reg->var_off) && amount < 64) + scalar_step_scale(dst_reg, 1ULL << amount); + else + reg_step_reset(dst_reg); +} + /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. * Caller should also handle BPF_MOV case separately. * If we return -EACCES, caller may want to try again treating pointer as a @@ -14589,8 +16299,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn u32 dst = insn->dst_reg; const char *reason; int ret, bounds_ret; + bool to_scalar; dst_reg = ®s[dst]; + /* + * Only += and -= keep a pointer. Any other op makes a number of it, + * which is fine when the program may see values of pointers anyway. + */ + to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks; if ((known && (smin_val != smax_val || umin_val != umax_val)) || smin_val > smax_val || umin_val > umax_val) { @@ -14601,7 +16317,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return 0; } - if (BPF_CLASS(insn->code) != BPF_ALU64) { + if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) { /* 32-bit ALU ops on pointers produce (meaningless) scalars */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { __mark_reg_unknown(env, dst_reg); @@ -14667,6 +16383,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EACCES; } + if (to_scalar) { + __mark_reg_unknown(env, dst_reg); + return 0; + } + /* For 'scalar += pointer', dst_reg inherits the complete pointer * register state. Individual fields may be adjusted later by pointer * arithmetic. Callers guarantee that below does not overwrite off_reg. @@ -14726,6 +16447,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ + scalar_step_add(dst_reg, ptr_reg, off_reg); dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; @@ -14787,6 +16509,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn if ((!known && smin_val < 0) || dst_reg->range < 0) memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); } + reg_step_reset(dst_reg); break; case BPF_AND: case BPF_OR: @@ -15494,6 +17217,7 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins * Bounds will be re-derived from the new tnum later. */ __mark_reg_unbounded(dst_reg); + reg_step_reset(dst_reg); } /* For bswap16/32, truncate dst register to match the swapped size */ if (insn->imm == 16 || insn->imm == 32) @@ -15620,6 +17344,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, */ switch (opcode) { case BPF_ADD: + scalar_step_add(dst_reg, dst_reg, &src_reg); scalar32_min_max_add(dst_reg, &src_reg); scalar_min_max_add(dst_reg, &src_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); @@ -15628,6 +17353,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &src_reg); scalar_min_max_sub(dst_reg, &src_reg); dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); + reg_step_reset(dst_reg); break; case BPF_NEG: env->fake_reg[0] = *dst_reg; @@ -15635,8 +17361,10 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &env->fake_reg[0]); scalar_min_max_sub(dst_reg, &env->fake_reg[0]); dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off); + reg_step_reset(dst_reg); break; case BPF_MUL: + scalar_step_mul(dst_reg, &src_reg); dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); scalar32_min_max_mul(dst_reg, &src_reg); scalar_min_max_mul(dst_reg, &src_reg); @@ -15657,6 +17385,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_sdiv(dst_reg, &src_reg); else scalar_min_max_udiv(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_MOD: /* BPF mod specification: x % 0 = x */ @@ -15672,6 +17401,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_smod(dst_reg, &src_reg); else scalar_min_max_umod(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { @@ -15682,6 +17412,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { @@ -15692,13 +17423,16 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_XOR: dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); scalar32_min_max_xor(dst_reg, &src_reg); scalar_min_max_xor(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_LSH: + scalar_step_lsh(dst_reg, &src_reg); if (alu32) scalar32_min_max_lsh(dst_reg, &src_reg); else @@ -15709,15 +17443,18 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_rsh(dst_reg, &src_reg); else scalar_min_max_rsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_ARSH: if (alu32) scalar32_min_max_arsh(dst_reg, &src_reg); else scalar_min_max_arsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_END: scalar_byte_swap(dst_reg, insn); + reg_step_reset(dst_reg); break; default: break; @@ -15929,6 +17666,112 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, return 0; } +/* + * Many checks done by this function are quite conservative. + * This is because main verification pass does not maintain + * enough information to track object identities for some + * of the interesting types, e.g. PTR_TO_MEM. + */ +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b) +{ + if (reg_a == reg_b) + return true; + /* Require matching flags and base types. */ + if (reg_a->type != reg_b->type) + return false; + /* NULL can't be compared to some base+offset pointer. */ + if (type_may_be_null(reg_a->type)) + return false; + + switch (base_type(reg_a->type)) { + case PTR_TO_STACK: + return reg_a->frameno == reg_b->frameno; + case PTR_TO_MAP_VALUE: + if (reg_a->map_ptr != reg_b->map_ptr || reg_a->map_uid != reg_b->map_uid) + return false; + if (reg_a->id && reg_a->id == reg_b->id) + return true; + /* + * A plain single-element array has one stable value address, + * but not when BPF_F_INNER_MAP is set, see arraymap.c:array_map_meta_equal(). + */ + if (reg_a->map_ptr->map_type == BPF_MAP_TYPE_ARRAY && + reg_a->map_ptr->max_entries == 1 && + !(reg_a->map_ptr->map_flags & BPF_F_INNER_MAP)) + return true; + /* + * The rules above can be simplified / relaxed if: + * - fresh IDs would always be assigned for map-value lookups; + * - direct map value loads would always have and ID of zero. + */ + return false; + case PTR_TO_MEM: + case PTR_TO_BUF: + return reg_a->id && reg_a->id == reg_b->id; + + default: + return false; + } +} + +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step) +{ + reg->r64 = r64; + reg->r32 = r32; + reg->step = step; + reg->base = base; + reg->var_off = tnum_unknown; + reg_bounds_sync(reg); /* this should infer the tnum alignment */ + return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); +} + +/* + * Checks if pointer and packet metadata matches between two registers. + * For use with bpf_reg_union(). + */ +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b) +{ + if (a->type != b->type) + return false; + if (a->type == SCALAR_VALUE) + return true; + return !memcmp(a, b, offsetof(struct bpf_reg_state, var_off)) && + a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid; +} + +/* + * Extends acc's scalar range to include src's scalar range. + * Caller must ensure that other kinds of metadata is compatible. + * Caller must clear acc's scalar ID. + */ +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src) +{ + u16 base, step; + + if (!bpf_reg_union_compatible(acc, src)) { + verifier_bug(env, "union of incompatible registers"); + return -EFAULT; + } + acc->r64 = cnum64_union(acc->r64, src->r64); + acc->r32 = cnum32_union(acc->r32, src->r32); + acc->var_off = tnum_union(acc->var_off, src->var_off); + + /* Retain a common congruence if the bases agree modulo the gcd. */ + step = gcd(acc->step, src->step); + base = acc->base % step; + if (base != src->base % step) { + reg_step_reset(acc); + } else { + acc->base = base; + acc->step = step; + } + reg_bounds_sync(acc); + return reg_bounds_sanity_check(env, acc, "bpf_reg_union"); +} + /* check validity of 32-bit and 64-bit arithmetic operations */ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) { @@ -16198,8 +18041,6 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32); -static u8 rev_opcode(u8 opcode); - /* * Learn more information about live branches by simulating refinement on both branches. * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means @@ -16208,7 +18049,7 @@ static u8 rev_opcode(u8 opcode); static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32) { /* Fallthrough (FALSE) branch */ - regs_refine_cond_op(&env->false_reg1, &env->false_reg2, rev_opcode(opcode), is_jmp32); + regs_refine_cond_op(&env->false_reg1, &env->false_reg2, bpf_rev_opcode(opcode), is_jmp32); reg_bounds_sync(&env->false_reg1); reg_bounds_sync(&env->false_reg2); /* @@ -16394,7 +18235,7 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s return simulate_both_branches_taken(env, opcode, is_jmp32); } -static int flip_opcode(u32 opcode) +int bpf_flip_opcode(u32 opcode) { /* How can we transform "a <op> b" into "b <op> a"? */ static const u8 opcode_flip[16] = { @@ -16425,7 +18266,7 @@ static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, pkt = dst_reg; } else if (dst_reg->type == PTR_TO_PACKET_END) { pkt = src_reg; - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); } else { return -1; } @@ -16480,7 +18321,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); } /* and ensure that reg2 is a constant */ @@ -16514,7 +18355,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* Opcode that corresponds to a *false* branch condition. * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 */ -static u8 rev_opcode(u8 opcode) +u8 bpf_rev_opcode(u8 opcode) { switch (opcode) { case BPF_JEQ: return BPF_JNE; @@ -16549,7 +18390,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state case BPF_JGT: case BPF_JSGE: case BPF_JSGT: - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); break; default: @@ -16607,7 +18448,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor * bit 1 is set, which we can readily use in adjustments. */ - if (!is_power_of_2(val)) + if (hweight64(val) != 1) break; if (is_jmp32) { t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); @@ -16616,7 +18457,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); } break; - case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ + case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see bpf_rev_opcode() */ if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) @@ -16626,6 +18467,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * violations if we're on a dead branch. */ __mark_reg_unbounded(reg1); + reg_step_reset(reg1); if (is_jmp32) { t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); @@ -16893,10 +18735,12 @@ static void collect_linked_regs(struct bpf_verifier_env *env, reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; __collect_linked_regs(linked_regs, reg, id, i, j, false); } } @@ -16916,7 +18760,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s for (i = 0; i < linked_regs->cnt; ++i) { e = &linked_regs->entries[i]; reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] - : &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr; + : &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr; if (reg->type != SCALAR_VALUE || reg == known_reg) continue; if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) @@ -16946,6 +18790,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s reg->delta = saved_off; reg->id = saved_id; + scalar_step_add(reg, reg, &fake_reg); scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); @@ -16981,6 +18826,23 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return -EINVAL; } + if (bpf_is_unwind_goto(insn)) { + if (!env->cur_state->unwound) + return 0; + env->cur_state->unwound = false; + /* + * unwind_exit changed its return address, so the return is + * mispredicted and what follows unwind_goto runs speculatively. + */ + if (!env->bypass_spec_v1) { + err = sanitize_speculative_path(env, NULL, *insn_idx + 1, *insn_idx); + if (err) + return err; + } + *insn_idx += insn->off; + return 0; + } + if (opcode == BPF_JCOND) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; int idx = *insn_idx; @@ -17034,7 +18896,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } if (insn_flags) { - err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0); + err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0); if (err) return err; } @@ -17104,7 +18966,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (linked_regs.cnt > 1) { - err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); + u16 packed[LINKED_REGS_MAX]; + + linked_regs_pack(&linked_regs, packed); + err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt); if (err) return err; } @@ -17669,37 +19534,54 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; + + if (subprog_returns_void(env, subprog)) + return 0; + + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) @@ -17741,11 +19623,11 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, (bpf_verifier_inlines_helper_call(env, call->imm) || bpf_jit_inlines_helper_call(call->imm)); cs->is_void = fn->ret_type == RET_VOID; - cs->num_params = 0; + cs->arg_slot_cnt = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; - cs->num_params++; + cs->arg_slot_cnt++; } return true; } @@ -17757,7 +19639,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->num_params = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; @@ -17866,7 +19748,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, * - includes R1-R5 if corresponding parameter has is described * in the function prototype. */ - clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0); + clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0); /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS; @@ -18232,10 +20114,25 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env, return 0; } +static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target, + u32 subprog_start, u32 subprog_end) +{ + verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n", + env->insn_idx, target, subprog_start, subprog_end); + bpf_diag_program_structure( + env, env->insn_idx, "indirect jump leaves subprogram", + "Keep every reachable jump-table target inside the subprogram of the indirect jump.", + "Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).", + env->insn_idx, target, subprog_start, subprog_end); + return -EINVAL; +} + /* gotox *dst_reg */ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn) { struct bpf_verifier_state *other_branch; + struct bpf_subprog_info *subprog; + u32 subprog_start, subprog_end; struct bpf_reg_state *dst_reg; struct bpf_map *map; u32 min_index, max_index; @@ -18279,6 +20176,24 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in return -EINVAL; } + subprog = bpf_find_containing_subprog(env, env->insn_idx); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx)) + return -EFAULT; + subprog_start = subprog->start; + subprog_end = (subprog + 1)->start; + + /* + * Maps are confined to a subprog, see compute_subprog_jts(), and the + * targets are sorted and free of duplicates, see sort_insn_array_uniq(), + * so either all of them belong to this subprog or none of them does. + */ + if (env->gotox_tmp_buf->items[0] < subprog_start) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0], + subprog_start, subprog_end); + if (env->gotox_tmp_buf->items[n - 1] >= subprog_end) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1], + subprog_start, subprog_end); + for (i = 0; i < n - 1; i++) { mark_indirect_target(env, env->gotox_tmp_buf->items[i]); other_branch = push_stack(env, env->gotox_tmp_buf->items[i], @@ -18321,15 +20236,17 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg_type = cur_regs(env)[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; - err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -18344,11 +20261,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) env->jmps_processed++; if (opcode == BPF_CALL) { if (env->cur_state->active_locks) { - if ((insn->src_reg == BPF_REG_0 && - insn->imm != BPF_FUNC_spin_unlock && - insn->imm != BPF_FUNC_kptr_xchg) || - (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - !kfunc_spin_allowed(env, insn->imm, insn->off))) { + /* similar to static subprog calls callx is allowed under a lock */ + if (!bpf_is_callx(insn) && + ((insn->src_reg == BPF_REG_0 && + insn->imm != BPF_FUNC_spin_unlock && + insn->imm != BPF_FUNC_kptr_xchg) || + (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + !kfunc_spin_allowed(env, insn->imm, insn->off)))) { verbose(env, "function calls are not allowed while holding a lock\n"); bpf_diag_ctx_active( @@ -18361,6 +20280,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) mark_reg_scratched(env, BPF_REG_0); if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) cur_func(env)->no_stack_arg_load = true; + if (bpf_is_callx(insn)) + return check_func_callx(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) @@ -18401,6 +20322,178 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return -EFAULT; } +/* + * Push the loop entered at insn_idx onto the stack. Usually this adds a single + * entry on top of its enclosing loop. However, an edge may enter an inner loop + * directly, bypassing the header(s) of its enclosing loop(s), e.g.: + * + * 1: for (...): // enclosing loop, header at 1 + * 2: for (...): // inner loop, header at 2 + * ... + * 3: if ...: + * goto 2b; // enters loop 2 without going through header 1 + * + * Such a bypass makes the enclosing loop irreducible, so its header is missing + * from the stack and both headers (1) and (2) need to be pushed onto stack. + * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds; + * the bypassed ancestors are irreducible and pushed as non-terminating. + * + * Assumes loop_stack_pop() has already truncated the stack to the common ancestor + * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack. + */ +static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *loop_stack = frame->loop_stack; + u32 missing_headers[LOOP_STACK_SIZE]; + u32 cnt = frame->loop_stack_cnt; + u32 num_missing = 0; + int h; + + *pushed = false; + + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) { + if (cnt && loop_stack[cnt - 1].loop_id == h) + break; + if (num_missing == LOOP_STACK_SIZE) + goto e2big; + missing_headers[num_missing] = h; + } + + if (num_missing == 0) + return 0; + if (cnt + num_missing > LOOP_STACK_SIZE) + goto e2big; + frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing, + sizeof(*frame->loop_stack)); + if (!frame->loop_stack) + return -ENOMEM; + loop_stack = frame->loop_stack; + + for (; num_missing; num_missing--, cnt++) { + h = missing_headers[num_missing - 1]; + loop_stack[cnt] = (struct loop_stack_entry){ .loop_id = h }; + if (env->log.level & BPF_LOG_LEVEL2) + verbose(env, "entering loop %d\n", h); + *pushed = true; + } + frame->loop_stack_cnt = cnt; + return 0; + +e2big: + verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx); + return -E2BIG; +} + +/* + * An exit from a loop can cross several nested loops, e.g.: + * + * 1: for (...): + * 2: for (...): + * 3: for (...): + * if ...: // before goto the loop stack is [1, 2, 3 <top>] + * goto 1b; // after goto it should become [1] + */ +static int loop_stack_pop(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + int new_cnt = 0; + int h, i; + + /* + * Walk the loop nest of insn_idx outwards (innermost first) and stop at + * the first loop that is present on the stack - that loop is the common + * ancestor and becomes the new top. If no enclosing loop is on the + * stack, drop all loops. + */ + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) { + for (i = frame->loop_stack_cnt - 1; i >= 0; i--) { + if (frame->loop_stack[i].loop_id == h) { + new_cnt = i + 1; + goto pop; + } + } + } + +pop: + if (env->log.level & BPF_LOG_LEVEL2) + for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--) + verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id); + frame->loop_stack_cnt = new_cnt; + return 0; +} + +static int maybe_clamp_scev_regs(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + + if (frame->loop_stack_cnt == 0) { + verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx); + return -EFAULT; + } + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (!entry->terminates) + return 0; + + return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry->entry_state, + entry->max_header_count); +} + +static int handle_loop_entry_exit(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + struct bpf_loop_iters iters; + bool pushed, terminates; + int err; + + err = loop_stack_pop(env); + if (err) + return err; + + err = loop_stack_push(env, &pushed); + if (err) + return err; + + if (frame->loop_stack_cnt == 0) + return 0; + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (env->insn_idx != entry->loop_id) + return 0; + + /* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */ + if (!pushed) + return maybe_clamp_scev_regs(env); + + err = bpf_compute_loop_iters(env, env->cur_state, &iters); + if (err < 0) + return err; + + terminates = err == 1; + if (!terminates) + return 0; + + /* Create a loop_entry checkpoint before widening */ + err = bpf_split_cur_state(env); + if (err) + return err; + + entry->min_header_count = iters.min_header_count; + entry->max_header_count = iters.max_header_count; + entry->entry_state = cur->parent; + entry->terminates = true; + err = bpf_widen_scev_regs(env, cur, cur->parent, &iters); + if (err < 0) + return err; + return 0; +} + static int do_check(struct bpf_verifier_env *env) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); @@ -18415,6 +20508,12 @@ static int do_check(struct bpf_verifier_env *env) struct bpf_insn_aux_data *insn_aux; int err; + if (signal_pending(current)) + return -EAGAIN; + + if (need_resched()) + cond_resched(); + /* reset current history entry on each new instruction */ env->cur_hist_ent = NULL; @@ -18461,6 +20560,15 @@ static int do_check(struct bpf_verifier_env *env) } } + /* + * Possibly widen the registers before creating a checkpoint + * in bpf_is_state_visited(). The next loop iteration will + * have a chance to hit this checkpoint and converge. + */ + err = handle_loop_entry_exit(env); + if (err) + return err; + if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); if (err < 0) @@ -18481,17 +20589,11 @@ static int do_check(struct bpf_verifier_env *env) } if (bpf_is_jmp_point(env, env->insn_idx)) { - err = bpf_push_jmp_history(env, state, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0); if (err) return err; } - if (signal_pending(current)) - return -EAGAIN; - - if (need_resched()) - cond_resched(); - if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, @@ -18501,6 +20603,13 @@ static int do_check(struct bpf_verifier_env *env) do_print_state = false; } + if (bpf_loop_at_index(env, env->insn_idx) >= 0 && + cur_func(env)->loop_stack_cnt == 0) { + verifier_bug(env, "loop stack empty at %d, while inside the loop %d\n", + env->insn_idx, bpf_loop_at_index(env, env->insn_idx)); + return -EFAULT; + } + if (env->log.level & BPF_LOG_LEVEL) { if (verifier_state_scratched(env)) print_insn_state(env, state, state->curframe); @@ -18833,12 +20942,6 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, { enum bpf_prog_type prog_type = resolve_prog_type(prog); - if (map->excl_prog_sha && - memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) { - verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); - return -EACCES; - } - if (btf_record_has_field(map->record, BPF_LIST_HEAD) || btf_record_has_field(map->record, BPF_RB_ROOT)) { if (is_tracing_prog_type(prog_type)) { @@ -18872,35 +20975,11 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EINVAL; } - if (prog->sleepable) - switch (map->map_type) { - case BPF_MAP_TYPE_HASH: - case BPF_MAP_TYPE_RHASH: - case BPF_MAP_TYPE_LRU_HASH: - case BPF_MAP_TYPE_ARRAY: - case BPF_MAP_TYPE_PERCPU_HASH: - case BPF_MAP_TYPE_PERCPU_ARRAY: - case BPF_MAP_TYPE_LRU_PERCPU_HASH: - case BPF_MAP_TYPE_LPM_TRIE: - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - case BPF_MAP_TYPE_RINGBUF: - case BPF_MAP_TYPE_USER_RINGBUF: - case BPF_MAP_TYPE_INODE_STORAGE: - case BPF_MAP_TYPE_SK_STORAGE: - case BPF_MAP_TYPE_TASK_STORAGE: - case BPF_MAP_TYPE_CGRP_STORAGE: - case BPF_MAP_TYPE_QUEUE: - case BPF_MAP_TYPE_STACK: - case BPF_MAP_TYPE_ARENA: - case BPF_MAP_TYPE_INSN_ARRAY: - case BPF_MAP_TYPE_PROG_ARRAY: - break; - default: - verbose(env, - "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); - return -EINVAL; - } + if (prog->sleepable && !map_needs_no_rcu(map)) { + verbose(env, + "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); + return -EINVAL; + } if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) { @@ -18936,6 +21015,85 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return 0; } +/* + * Keep track of whether the map is used by one program only. Such program may + * store the addresses of its functions into the map when it's frozen, see + * resolve_func_ptrs(), since nothing else relies on what the map has. After + * that the map is not available to other programs. + */ +static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) +{ + unsigned long me = (unsigned long)env->prog->aux, old; + + for (;;) { + old = READ_ONCE(map->user); + if (old == me || old == BPF_MAP_USER_MANY) + return 0; + if (old & BPF_MAP_USER_PATCHED) { + verbose(env, "map '%s' has addresses of functions of another program\n", + map->name); + return -EBUSY; + } + if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old) + return 0; + } +} + +/* + * An exclusive map is bound to the digest of its program. The digest covers + * the insns as they are verified, after the CO-RE relocations the kernel + * applies, so it exists only once check_and_resolve_insns() has computed it, + * and the maps are checked against it there, whether an insn referred to + * them or a continuous fd_array bound them before. + */ +static int check_excl_maps(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + if (map->excl_prog_sha && + memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) { + verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); + return -EACCES; + } + } + return 0; +} + +/* + * The program may use every map it referred to, see check_excl_maps(). Only now + * does it leave a trace on them: it claims each as its user, see + * bpf_map_claim(), and takes over its insn arrays. A program rejected before + * this point, one that is not the owner of an exclusive map in particular, + * leaves the maps as it found them. Nothing consults a claim before + * resolve_func_ptrs() and the fixup passes, which run later. + */ +static int claim_used_maps(struct bpf_verifier_env *env) +{ + int i, err; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + err = bpf_map_claim(env, map); + if (err) + return err; + + if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY) + continue; + err = bpf_insn_array_init(map, env->prog); + if (err) { + verbose(env, "Failed to properly initialize insn array\n"); + return err; + } + env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; + } + return 0; +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -18961,7 +21119,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (err) return err; - if (env->prog->sleepable) + if (bpf_prog_runs_sleepable(env->prog)) atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, @@ -18973,16 +21131,6 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - err = bpf_insn_array_init(map, env->prog); - if (err) { - verbose(env, "Failed to properly initialize insn array\n"); - return err; - } - env->insn_array_maps[env->insn_array_map_cnt++] = map; - env->prog->jit_required = true; - } - return env->used_map_cnt - 1; } @@ -19127,6 +21275,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) switch (opcode) { case BPF_CALL: + if (bpf_is_callx(insn)) { + /* callx dst_reg */ + if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { + verbose(env, "BPF_CALL|BPF_X uses reserved fields\n"); + return -EINVAL; + } + return 0; + } if (BPF_SRC(insn->code) != BPF_K || (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && @@ -19151,14 +21307,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; case BPF_EXIT: if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 || - insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 || + insn->src_reg > BPF_UNWIND_EXIT || insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) { verbose(env, "BPF_EXIT uses reserved fields\n"); return -EINVAL; } return 0; case BPF_JCOND: - if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg != BPF_MAY_GOTO || + if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg > BPF_UNWIND_GOTO || insn->dst_reg || insn->imm) { verbose(env, "invalid may_goto imm %d\n", insn->imm); return -EINVAL; @@ -19378,6 +21534,107 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) * 'struct bpf_map *' into a register instead of user map_fd. * These pointers will be used later by verifier to validate map access. */ + err = check_excl_maps(env); + if (err) + return err; + return claim_used_maps(env); +} + +/* that many pointers to functions are recognized in the maps of a program at most */ +#define BPF_MAX_FUNC_PTRS (1 << 20) + +static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off, + u32 xlated_off) +{ + struct bpf_func_ptr *ptrs; + + if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) { + verbose(env, "too many pointers to functions in maps, the limit is %d\n", + BPF_MAX_FUNC_PTRS); + return -E2BIG; + } + + /* grow by doubling, the array is sorted and searched later */ + if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) { + ptrs = kvrealloc(env->func_ptrs, + array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)), + GFP_KERNEL_ACCOUNT); + if (!ptrs) + return -ENOMEM; + env->func_ptrs = ptrs; + } + env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){ + .map = map, + .map_off = map_off, + .orig_off = xlated_off, + .xlated_off = xlated_off, + }; + return 0; +} + +/* + * Compilers put pointers to functions into read-only data: tables of functions, + * structures of operations, vtables, where they are mixed with other data. + * The loader stores such data in a frozen read-only array map and resolves + * a pointer to a static function to the offset in bytes of its first + * instruction in the program: the address of the function in the program. + * + * Find 64-bit values that look like that in the maps of a program that uses + * callx. It's a guess. When the value is not a pointer, the program either + * fails to load, because it does with a pointer what can be done with + * a number only, or it sees the address of a function instead of the number. + * It's known before the control flow graph of the program is built and the main + * verification pass begins which functions may be called via callx. + * + * When the program is jitted the offsets are replaced with the addresses of + * the functions in the map itself, see jit_subprogs(). Hence the program has to + * be the only user of the map, see bpf_map_claim(): nothing else may rely on + * what the map had. + * + * The program reads the addresses of its functions from there like any other + * data, so it has to be allowed to leak pointers. + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env) +{ + int insn_cnt = env->prog->len; + int i, err, subprog; + struct bpf_map *map; + u64 addr, val; + u32 off; + + if (!env->has_callx || !env->allow_ptr_leaks) + return 0; + + for (i = 0; i < env->used_map_cnt; i++) { + map = env->used_maps[i]; + /* coincidences in maps that are shared with other programs don't matter */ + if (READ_ONCE(map->user) != (unsigned long)env->prog->aux) + continue; + if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 || + !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || + !IS_ERR_OR_NULL(map->record)) + continue; + if (map->ops->map_direct_value_addr(map, &addr, 0)) + continue; + + for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) { + if (!(off % SZ_64K)) + cond_resched(); + val = *(u64 *)(unsigned long)(addr + off); + if (!val || val % sizeof(struct bpf_insn) || + val / sizeof(struct bpf_insn) >= insn_cnt) + continue; + subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn)); + if (subprog <= 0 || bpf_subprog_is_global(env, subprog)) + continue; + err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn)); + if (err) + return err; + } + } + if (env->func_ptr_cnt) + sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs), + cmp_func_ptrs, NULL); return 0; } @@ -19488,7 +21745,8 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable) +static int do_check_common(struct bpf_verifier_env *env, int subprog, + enum bpf_func_context context) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); @@ -19508,7 +21766,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = is_sleepable; + state->in_sleepable = context == BPF_FUNC_CTX_SLEEPABLE; + env->no_rcu_read_lock = context == BPF_FUNC_CTX_NO_RCU_LOCK; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -19548,25 +21807,30 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl } /* Also ensure the callback only has a single scalar argument. */ - if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { + if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } - for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = BPF_REG_1; + i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i]; if (arg->arg_type == ARG_PTR_TO_CTX) { reg->type = PTR_TO_CTX; mark_reg_known_zero(env, regs, i); - } else if (arg->arg_type == ARG_ANYTHING) { + } else if (arg->arg_type == ARG_SCALAR) { reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - /* assume unspecial LOCAL dynptr type */ - __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0); + /* + * The callers may hand in a dynptr of any type. + * Take it for that of an skb, so that a write + * through it and its slices are treated with care. + */ + __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { reg->type = PTR_TO_MEM; reg->type |= arg->arg_type & @@ -19597,7 +21861,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl goto out; } } - if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { + if (env->prog->type == BPF_PROG_TYPE_EXT && + sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) { verbose(env, "freplace programs with >%d args not supported yet\n", MAX_BPF_FUNC_REG_ARGS); ret = -EINVAL; @@ -19610,9 +21875,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); - if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { + if (ret || sub->arg_slot_cnt != 1 || + sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; - sub->arg_cnt = 1; + sub->arg_slot_cnt = 1; sub->stack_arg_cnt = 0; } } @@ -19722,7 +21988,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) /* * We can't loop forever as each pass verifies at least one new context, - * and there are only two contexts per global subprog. + * and there are only three contexts per global subprog. */ if (new_cnt) goto again; @@ -20108,6 +22374,109 @@ static bool attach_uses_trampoline_retval(enum bpf_attach_type type) } } +/* + * BPF code passes argument slot N where the kernel convention passes its Nth + * u64 argument, so describe a BPF function to its trampoline as one u64 per + * slot, and every arch places its arguments right. + */ +static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname, + struct btf_func_model *m) +{ + u32 i, nslots = 0; + + for (i = 0; i < m->nr_args; i++) + nslots += btf_func_model_arg_slots(m, i); + if (nslots > MAX_BPF_FUNC_ARGS) { + bpf_log(log, "The function %s has %u argument slots. Too many.\n", + tname, nslots); + return -EINVAL; + } + + for (i = 0; i < nslots; i++) { + m->arg_size[i] = sizeof(u64); + m->arg_flags[i] = 0; + } + m->nr_args = nslots; + return 0; +} + +/* + * The verifier lets a static function use stack arguments counted from its + * BTF, even if that BTF turns out unreliable. A trampoline on the function is + * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the + * stack arguments, so such a function cannot be attached to. + */ +static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf, + const struct btf_type *func_proto, const char *tname) +{ + struct btf_func_model m; + u32 i, nslots = 0; + + if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) { + /* at least a slot for each argument */ + nslots = btf_type_vlen(func_proto); + } else { + for (i = 0; i < m.nr_args; i++) + nslots += btf_func_model_arg_slots(&m, i); + } + if (nslots <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n", + tname); + return -EINVAL; +} + +/* Functions may share a type id. The first of them is attached to. */ +static int find_tgt_subprog(const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int i; + + for (i = 0; i < aux->func_info_cnt; i++) + if (aux->func_info[i].type_id == btf_id) + return i; + + return -1; +} + +/* + * The prologue of an extension gives a cloned skb a new head, unless the + * prologue of the target program has done that. The callers of the function + * that it replaces must not hold on to the old head. + */ +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int subprog = find_tgt_subprog(tgt_prog, btf_id); + + if (aux->has_prologue) + return 0; + + if (subprog >= 0 && !aux->func_info_aux[subprog].called_with_pkt_ptrs) + return 0; + + bpf_log(log, + "Extension writing packet data cannot replace a function whose callers keep packet pointers\n"); + return -EINVAL; +} + +/* + * An extension is verified as a program that runs under the RCU read lock. Does + * the function that it is to replace? + */ +static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog) +{ + const struct bpf_func_info_aux *sub_aux = &prog->aux->func_info_aux[subprog]; + + if (!subprog) + return !prog->sleepable; + + return !sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] && + !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK]; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -20119,7 +22488,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, char trace_symbol[KSYM_SYMBOL_LEN]; const char prefix[] = "btf_trace_"; struct bpf_raw_event_map *btp; - int ret = 0, subprog = -1, i; + int ret = 0, subprog = -1; const struct btf_type *t; bool conservative = true; const char *tname, *fname; @@ -20151,6 +22520,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, struct bpf_prog_aux *aux = tgt_prog->aux; bool tgt_changes_pkt_data; bool tgt_might_sleep; + bool tgt_can_unwind; if (bpf_prog_is_dev_bound(prog->aux) && !bpf_prog_dev_bound_match(prog, tgt_prog)) { @@ -20158,11 +22528,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, return -EINVAL; } - for (i = 0; i < aux->func_info_cnt; i++) - if (aux->func_info[i].type_id == btf_id) { - subprog = i; - break; - } + subprog = find_tgt_subprog(tgt_prog, btf_id); if (subprog == -1) { bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; @@ -20183,9 +22549,20 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog_extension ? "Extension" : "Tracing"); return -EINVAL; } + /* unwind_exit of the main prog is exit, unless it's an extension */ + tgt_can_unwind = (subprog || tgt_prog->type == BPF_PROG_TYPE_EXT) && + (aux->func ? aux->func[subprog]->aux->can_unwind + : aux->can_unwind); + /* unwind_exit would return to the trampoline that called the target */ + if (tgt_can_unwind && !prog_extension && + prog->expected_attach_type != BPF_TRACE_FENTRY) { + bpf_log(log, "Only fentry can attach to %s, which can unwind\n", tname); + return -EINVAL; + } conservative = aux->func_info_aux[subprog].unreliable; if (prog_extension) { - if (conservative) { + if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL || + conservative) { bpf_log(log, "Cannot replace static functions\n"); return -EINVAL; @@ -20195,6 +22572,20 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension programs should be JITed\n"); return -EINVAL; } + /* + * How much of the buffer of the tracepoint is used + * is checked for the program that is attached to it. + */ + if (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { + bpf_log(log, + "Cannot extend a writable raw tracepoint program\n"); + return -EINVAL; + } + if (!func_has_rcu_read_lock(tgt_prog, subprog)) { + bpf_log(log, + "Cannot extend a function that runs without RCU read lock\n"); + return -EINVAL; + } tgt_changes_pkt_data = aux->func ? aux->func[subprog]->aux->changes_pkt_data : aux->changes_pkt_data; @@ -20203,6 +22594,9 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program changes packet data, while original does not\n"); return -EINVAL; } + if (prog->aux->has_prologue && + bpf_check_ext_prologue(log, tgt_prog, btf_id)) + return -EINVAL; tgt_might_sleep = aux->func ? aux->func[subprog]->aux->might_sleep @@ -20212,6 +22606,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program may sleep, while original does not\n"); return -EINVAL; } + /* There is no unwind_goto after the calls of the original */ + if (prog->aux->can_unwind && !tgt_can_unwind) { + bpf_log(log, + "Extension program can unwind, while original does not\n"); + return -EINVAL; + } } if (!tgt_prog->jited) { bpf_log(log, "Can attach to only JITed progs\n"); @@ -20375,12 +22775,23 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) return -EINVAL; - if (tgt_prog && conservative) + if (tgt_prog && conservative) { + if (subprog > 0) { + ret = check_unreliable_func_args(log, btf, t, tname); + if (ret) + return ret; + } t = NULL; + } ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) return ret; + if (tgt_prog) { + ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel); + if (ret < 0) + return ret; + } if (tgt_info->fmodel.ret_size > 8 && attach_uses_trampoline_retval(prog->expected_attach_type)) { @@ -20857,6 +23268,66 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc return 0; } +static int add_kfunc_desc_target(struct bpf_verifier_env *env, + const struct bpf_kfunc_desc *target_desc, + u16 base_desc_idx, u16 *desc_idx) +{ + struct bpf_kfunc_desc desc = *target_desc; + struct bpf_kfunc_desc_tab *new_tab; + struct bpf_kfunc_desc_tab *tab; + struct bpf_prog_aux *prog_aux; + u32 i; + + prog_aux = env->prog->aux; + tab = prog_aux->kfunc_tab; + + if (base_desc_idx >= tab->nr_base_descs) { + verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)", + base_desc_idx, tab->nr_base_descs); + return -EFAULT; + } + if (tab->descs[base_desc_idx].func_id != desc.func_id) { + verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id); + return -EFAULT; + } + if (tab->descs[base_desc_idx].offset != desc.offset) { + verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset); + return -EFAULT; + } + + if (tab->descs[base_desc_idx].addr == desc.addr) { + *desc_idx = base_desc_idx; + return 0; + } + + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) { + if (tab->descs[i].func_id == desc.func_id && + tab->descs[i].offset == desc.offset && + tab->descs[i].addr == desc.addr) { + *desc_idx = i; + return 0; + } + } + + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { + verbose(env, "too many different kernel function call targets\n"); + return -E2BIG; + } + + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), + GFP_KERNEL_ACCOUNT); + if (!new_tab) + return -ENOMEM; + tab = new_tab; + prog_aux->kfunc_tab = tab; + + *desc_idx = tab->nr_descs; + tab->descs[tab->nr_descs++] = desc; + return 0; +} + static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, @@ -20877,7 +23348,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { + struct bpf_kfunc_desc desc_copy; struct bpf_kfunc_desc *desc; + unsigned long call_imm; + u16 base_desc_idx, desc_idx; + bool near_call; int err; if (!insn->imm) { @@ -20897,13 +23372,30 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn->imm); return -EFAULT; } + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs; + + near_call = !bpf_jit_supports_far_kfunc_call(); + desc_copy = *desc; + desc = &desc_copy; err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) - insn->imm = BPF_CALL_IMM(desc->addr); + if (near_call) { + call_imm = BPF_CALL_IMM(desc->addr); + if ((unsigned long)(s32)call_imm != call_imm) { + verbose(env, "address of kernel func_id %u is out of range\n", + desc->func_id); + return -EINVAL; + } + insn->imm = call_imm; + } + + err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx); + if (err) + return err; + insn->off = desc_idx; if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; @@ -21063,6 +23555,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +/* + * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably + * above the largest signature the kernel can verify, and far below anything + * that would make rejecting a load expensive. Deliberately a fixed number so + * that what the syscall accepts does not depend on PAGE_SIZE. + */ +#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) + static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) { switch (keyring_id) { @@ -21072,6 +23572,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21101,17 +23603,33 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, int err = 0; /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. + * The signature covers the insns as submitted. CO-RE relocations would + * have the kernel rewrite them afterwards, with values the loader picks + * through the BTF it supplies, see bpf_check_core_relo(). */ + if (attr->core_relo_cnt) { + verbose(env, "signed program cannot carry CO-RE relocations\n"); + return -EACCES; + } if (!attr->signature_size || - attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + key = bpf_lookup_keyring(); + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; @@ -21203,6 +23721,65 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, return err; } +/* + * bpf_clone_redirect() transmits from inside the program, so a tc egress or + * lwt_xmit program that redirects to its own device runs again on top of its + * own frame, and again from there, until the datapath's recursion limit drops + * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack + * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program + * calling it keeps the smaller budget. The redirect helpers that transmit + * after the program has returned leave no frame behind. + */ +static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + int i; + + for (i = 0; i < prog->len; i++, insn++) + if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect) + return true; + return false; +} + +/* Various log level 2 information about the program */ +static void log_program(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *insn_aux = env->insn_aux_data; + struct bpf_insn *insns = env->prog->insnsi; + u32 insn_cnt = env->prog->len; + u64 pos, insn_pos; + u32 i, j; + + verbose(env, "Program dump (scc? loop_header? idom insn#: live_regs_before):\n"); + for (i = 0; i < insn_cnt; ++i) { + verbose_linfo(env, i, " ; "); + if (env->insn_aux_data[i].scc) + verbose(env, "%3d ", env->insn_aux_data[i].scc); + else + verbose(env, " "); + if (env->insn_aux_data[i].loop_header >= 0) + verbose(env, "%3d ", env->insn_aux_data[i].loop_header); + else + verbose(env, " "); + verbose(env, "%3d ", env->idoms[i]); + verbose(env, "%3d: ", i); + for (j = BPF_REG_0; j < BPF_REG_10; ++j) + if (insn_aux[i].live_regs_before & BIT(j)) + verbose(env, "%d", j); + else + verbose(env, "."); + verbose(env, " "); + pos = env->log.end_pos; + bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); + verbose(env, "\n"); + if (bpf_is_ldimm64(&insns[i])) + i++; + } +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -21227,6 +23804,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bt.env = env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; + env->stack_limit = bpf_prog_stack_limit(env->prog); + if (bpf_prog_reenters_datapath(env->prog)) + env->stack_limit = MAX_BPF_STACK; env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); @@ -21249,11 +23829,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = -EINVAL; goto err_prep; } - if (env->signature) { - ret = bpf_prog_calc_tag(env->prog); - if (ret < 0) - goto err_prep; - } ret = process_fd_array(env, attr, uattr); if (ret) @@ -21307,6 +23882,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; + env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), @@ -21329,22 +23905,29 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Discover all subprograms before validating their layout and BTF. */ + /* Discover all subprograms and collect the properties needed by BTF validation. */ ret = add_subprogs(env); if (ret < 0) goto skip_full_check; + find_subprog_properties(env); + + /* Validate BTF before reporting subprogram layout errors. */ + ret = bpf_check_btf_info(env, attr, uattr); + if (ret < 0) + goto skip_full_check; + ret = check_subprogs(env); if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout. */ - ret = bpf_check_btf_info(env, attr, uattr); + /* Validate instructions and resolve the program's referenced resources. */ + ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check; - /* Validate instructions and resolve the program's referenced resources. */ - ret = check_and_resolve_insns(env); + /* Find pointers to functions in the read-only maps of the program. */ + ret = resolve_func_ptrs(env); if (ret < 0) goto skip_full_check; @@ -21363,6 +23946,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + ret = check_unwind_insns(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_postorder(env); if (ret < 0) goto skip_full_check; @@ -21391,10 +23978,38 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + + prepare_subprog_args(env); + + ret = bpf_compute_idoms(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_loops(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; + if (env->log.level & BPF_LOG_LEVEL2) + log_program(env); + + ret = bpf_init_scev(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_scev(env); + if (ret < 0) + goto skip_full_check; + + bpf_free_subprog_jts(env); + ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check; @@ -21402,6 +24017,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = do_check_main(env); ret = ret ?: do_check_subprogs(env); + /* reject recursion through the callx edges found by the main pass */ + if (ret == 0 && env->callx_edges) + ret = sort_subprogs_topo(env); + if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env); @@ -21535,16 +24154,22 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, release_maps(env); release_btfs(env); err_free_env: + bpf_free_subprog_jts(env); if (env->insn_aux_data) bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); + bpf_free_scev(env); bpf_stack_liveness_free(env); + kvfree(env->cfg.postorder_nums); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + kvfree(env->callx_edges); + kvfree(env->func_ptrs); bpf_diag_free(env); + kvfree(env->idoms); kvfree(env); return ret; }
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 6609346..90937dc 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c
@@ -451,8 +451,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) * * This function releases memory allocated by dma_alloc_contiguous(). As the * cma_release returns false when provided pages do not belong to contiguous - * area and true otherwise, this function then does a fallback __free_pages() - * upon a false-return. + * area and true otherwise, this function then does a fallback + * free_pages_exact() upon a false-return. */ void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { @@ -476,7 +476,7 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size) } /* not in any cma, free from buddy */ - __free_pages(page, get_order(size)); + free_pages_exact(page_address(page), size); } /*
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 026d50d..36c1f7c 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c
@@ -489,8 +489,10 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, /* If the device is not writing memory then we don't have any * concerns about the cpu consuming stale data. This mitigates * legitimate usages of overlapping mappings. + * MMIO mappings are never CPU-cacheable and do not need tracking either. */ - if (entry->direction == DMA_TO_DEVICE) + if (entry->direction == DMA_TO_DEVICE || + entry->attrs & DMA_ATTR_MMIO) return 0; spin_lock_irqsave(&radix_lock, flags); @@ -519,7 +521,8 @@ static void active_cacheline_remove(struct dma_debug_entry *entry) unsigned long flags; /* ...mirror the insert case */ - if (entry->direction == DMA_TO_DEVICE) + if (entry->direction == DMA_TO_DEVICE || + entry->attrs & DMA_ATTR_MMIO) return; spin_lock_irqsave(&radix_lock, flags);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index da665ca..25e9a86 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c
@@ -120,6 +120,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, int node = dev_to_node(dev); struct page *page; u64 phys_limit; + void *va; WARN_ON_ONCE(!PAGE_ALIGNED(size)); @@ -133,9 +134,9 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, dma_free_contiguous(dev, page, size); } - while ((page = alloc_pages_node(node, gfp, get_order(size))) - && !dma_coherent_ok(dev, page_to_phys(page), size)) { - __free_pages(page, get_order(size)); + while ((va = alloc_pages_exact_nid(node, size, gfp)) && + !dma_coherent_ok(dev, virt_to_phys(va), size)) { + free_pages_exact(va, size); if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_limit < DMA_BIT_MASK(64) && @@ -146,6 +147,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, else return NULL; } + page = va ? virt_to_page(va) : NULL; return page; } @@ -179,6 +181,22 @@ static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size, return page; } +/* + * Check if the allocation can skip the kernel mapping and return the page + * pointer as the cookie. + */ +static bool dma_direct_use_no_mapping(struct device *dev, unsigned long attrs) +{ + if ((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) != + DMA_ATTR_NO_KERNEL_MAPPING) + return false; + if (is_swiotlb_for_alloc(dev)) + return false; + if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && !dev_is_dma_coherent(dev)) + return false; + return true; +} + static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp) { @@ -226,8 +244,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, if (attrs & DMA_ATTR_NO_WARN) gfp |= __GFP_NOWARN; - if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) == - DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) + if (dma_direct_use_no_mapping(dev, attrs)) return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); if (!dev_is_dma_coherent(dev)) { @@ -374,8 +391,7 @@ void dma_direct_free(struct device *dev, size_t size, if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) mark_mem_encrypted = true; - if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) == - DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) { + if (dma_direct_use_no_mapping(dev, attrs)) { /* cpu_addr is a struct page cookie, not a kernel address */ dma_free_contiguous(dev, cpu_addr, size); return;
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index bf2651a..82e7016 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c
@@ -1012,19 +1012,19 @@ size_t dma_max_mapping_size(struct device *dev) } EXPORT_SYMBOL_GPL(dma_max_mapping_size); -size_t dma_opt_mapping_size(struct device *dev) +size_t dma_max_opt_mapping_size(struct device *dev) { const struct dma_map_ops *ops = get_dma_ops(dev); size_t size = SIZE_MAX; if (use_dma_iommu(dev)) - size = iommu_dma_opt_mapping_size(); - else if (ops && ops->opt_mapping_size) - size = ops->opt_mapping_size(); + size = iommu_dma_max_opt_mapping_size(); + else if (ops && ops->max_opt_mapping_size) + size = ops->max_opt_mapping_size(); return min(dma_max_mapping_size(dev), size); } -EXPORT_SYMBOL_GPL(dma_opt_mapping_size); +EXPORT_SYMBOL_GPL(dma_max_opt_mapping_size); unsigned long dma_get_merge_boundary(struct device *dev) {
diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 6b5f920..5cdf3f60 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c
@@ -67,10 +67,13 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size, phys_addr_t phys; page = dma_alloc_contiguous(dev, size, gfp); - if (!page) - page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); - if (!page) - return NULL; + if (!page) { + void *va = alloc_pages_exact_nid(dev_to_node(dev), size, gfp); + + if (!va) + return NULL; + page = virt_to_page(va); + } phys = page_to_phys(page); if (use_dma_iommu(dev))
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 5edea93..2c57b49 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c
@@ -100,6 +100,7 @@ static struct io_tlb_mem io_tlb_default_mem; static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT; static unsigned long default_nareas; +static bool swiotlb_cmdline_size_set __initdata; /** * struct io_tlb_area - IO TLB memory area descriptor @@ -117,27 +118,20 @@ struct io_tlb_area { spinlock_t lock; }; -/* - * Round up number of slabs to the next power of 2. The last area is going - * be smaller than the rest if default_nslabs is not power of two. - * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE, - * otherwise a segment may span two or more areas. It conflicts with free - * contiguous slots tracking: free slots are treated contiguous no matter - * whether they cross an area boundary. - * - * Return true if default_nslabs is rounded up. - */ -static bool round_up_default_nslabs(void) +/* Return a power-of-two number of slabs that can be split between areas. */ +static unsigned long swiotlb_calc_nslabs(unsigned long size, + unsigned long nareas) { - if (!default_nareas) - return false; + unsigned long nslabs; - if (default_nslabs < IO_TLB_SEGSIZE * default_nareas) - default_nslabs = IO_TLB_SEGSIZE * default_nareas; - else if (is_power_of_2(default_nslabs)) - return false; - default_nslabs = roundup_pow_of_two(default_nslabs); - return true; + nslabs = ALIGN(DIV_ROUND_UP(size, IO_TLB_SIZE), IO_TLB_SEGSIZE); + if (nareas && nslabs < IO_TLB_SEGSIZE * nareas) + nslabs = IO_TLB_SEGSIZE * nareas; + + if (!is_power_of_2(nslabs)) + nslabs = roundup_pow_of_two(nslabs); + + return nslabs; } /** @@ -150,6 +144,8 @@ static bool round_up_default_nslabs(void) */ static void swiotlb_adjust_nareas(unsigned int nareas) { + unsigned long nslabs; + if (!nareas) nareas = 1; else if (!is_power_of_2(nareas)) @@ -158,9 +154,12 @@ static void swiotlb_adjust_nareas(unsigned int nareas) default_nareas = nareas; pr_info("area num %d.\n", nareas); - if (round_up_default_nslabs()) + nslabs = swiotlb_calc_nslabs(default_nslabs << IO_TLB_SHIFT, nareas); + if (nslabs != default_nslabs) { + default_nslabs = nslabs; pr_info("SWIOTLB bounce buffer size roundup to %luMB", (default_nslabs << IO_TLB_SHIFT) >> 20); + } } /** @@ -255,6 +254,7 @@ setup_io_tlb_npages(char *str) /* avoid tail segment of size < IO_TLB_SEGSIZE */ default_nslabs = ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE); + swiotlb_cmdline_size_set = true; } if (*str == ',') ++str; @@ -285,7 +285,7 @@ setup_io_tlb_npages(char *str) } early_param("swiotlb", setup_io_tlb_npages); -unsigned long swiotlb_size_or_default(void) +unsigned long swiotlb_default_pool_size(void) { return default_nslabs << IO_TLB_SHIFT; } @@ -297,13 +297,11 @@ void __init swiotlb_adjust_size(unsigned long size) * architectures such as those supporting memory encryption to * adjust/expand SWIOTLB size for their use. */ - if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT) + if (swiotlb_cmdline_size_set) return; - size = ALIGN(size, IO_TLB_SIZE); - default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE); - if (round_up_default_nslabs()) - size = default_nslabs << IO_TLB_SHIFT; + default_nslabs = swiotlb_calc_nslabs(size, default_nareas); + size = default_nslabs << IO_TLB_SHIFT; pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20); } @@ -2083,8 +2081,8 @@ static int __init rmem_swiotlb_setup(unsigned long node, of_get_flat_dt_prop(node, "no-map", NULL)) return -EINVAL; - pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n", - &rmem->base, (unsigned long)rmem->size / SZ_1M); + pr_info("Reserved memory: created restricted DMA pool at %pa, size %llu KiB\n", + &rmem->base, (unsigned long long)(rmem->size / SZ_1K)); return 0; }
diff --git a/kernel/kprobes.c b/kernel/kprobes.c index e787e494..2441b9e 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c
@@ -43,6 +43,7 @@ #include <linux/cleanup.h> #include <linux/wait.h> #include <linux/wait_bit.h> +#include <linux/rcupdate.h> #include <asm/sections.h> #include <asm/cacheflush.h> @@ -67,7 +68,7 @@ static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; /* NOTE: change this value only with 'kprobe_mutex' held */ static bool kprobes_all_disarmed; -/* This protects 'kprobe_table' and 'optimizing_list' */ +/* This protects 'kprobe_table' and 'optprobe_gens' */ static DEFINE_MUTEX(kprobe_mutex); static DEFINE_PER_CPU(struct kprobe *, kprobe_instance); @@ -476,18 +477,6 @@ bool kprobe_disarmed(struct kprobe *p) return kprobe_disabled(p) && list_empty(&op->list); } -/* Return true if the probe is queued on (un)optimizing lists */ -static bool kprobe_queued(struct kprobe *p) -{ - struct optimized_kprobe *op; - - if (kprobe_aggrprobe(p)) { - op = container_of(p, struct optimized_kprobe, kp); - if (!list_empty(&op->list)) - return true; - } - return false; -} /* * Return an optimized kprobe whose optimizing code replaces @@ -514,10 +503,39 @@ static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) return NULL; } -/* Optimization staging list, protected by 'kprobe_mutex' */ -static LIST_HEAD(optimizing_list); -static LIST_HEAD(unoptimizing_list); -static LIST_HEAD(freeing_list); +#define OPTPROBE_GEN_MAX 2 + +struct optprobe_generation { + /* Probes waiting for grace period to drain tasks from original text */ + struct list_head optimizing_list; + /* Probes queued for batch unoptimization upon dispatch */ + struct list_head unoptimizing_list; + /* In-use probes waiting for grace period to drain optinsn slot */ + struct list_head cooling_list; + /* Unused probes unhashed and waiting for batch release */ + struct list_head freeing_list; + /* Tasks RCU callback head for asynchronous waiting */ + struct rcu_head rcu; + /* True if dispatched and awaiting Tasks RCU callback */ + bool in_flight; + /* True when Tasks RCU callback has fired and ready to finalize */ + bool ready; +}; + +/* + * Generational ring of optprobes. + * + * Incoming probe requests are queued into the waiting room generation + * (optprobe_gens[optprobe_cur_gen]). When dispatched, the generation + * unoptimizes its probes, invokes call_rcu_tasks(), and optprobe_cur_gen + * advances to the next slot. + * + * To ensure an idle generation is always available to collect incoming + * requests without dynamic allocation, the last available generation slot + * is never dispatched until another generation has finished. + */ +static struct optprobe_generation optprobe_gens[OPTPROBE_GEN_MAX]; +static int optprobe_cur_gen; static void optimize_kprobe(struct kprobe *p); static struct task_struct *kprobe_optimizer_task; @@ -535,11 +553,66 @@ static unsigned long optimizer_passes; #define OPTIMIZE_DELAY 5 /* - * Optimize (replace a breakpoint with a jump) kprobes listed on - * 'optimizing_list'. + * Note: gen->cooling_list is not checked here because it is strictly + * an in-flight holding list for dispatched generations, so it is always + * empty in optprobe_cur_gen. */ -static void do_optimize_kprobes(void) +static bool optprobe_has_queued_probes(void) { + struct optprobe_generation *gen = &optprobe_gens[optprobe_cur_gen]; + + return !list_empty(&gen->optimizing_list) || + !list_empty(&gen->unoptimizing_list) || + !list_empty(&gen->freeing_list); +} + +static int optprobe_active_gens_count(void) +{ + int count = 0; + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + if (optprobe_gens[i].in_flight || optprobe_gens[i].ready) + count++; + } + return count; +} + +/* + * The last generation must not be fired until another generation is done. + * (Thus the last generation acts as the waiting room.) + */ +static bool optprobe_can_fire(void) +{ + return optprobe_active_gens_count() < OPTPROBE_GEN_MAX - 1; +} + +static bool optprobe_has_ready_gens(void) +{ + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + if (READ_ONCE(optprobe_gens[i].ready)) + return true; + } + return false; +} + +static bool optprobe_optimizer_busy(void) +{ + return optprobe_has_queued_probes() || (optprobe_active_gens_count() > 0); +} + +/* + * Unoptimize (replace a jump with a breakpoint and remove the breakpoint + * if need) kprobes listed on 'unopt_list'. + */ +static void do_unoptimize_kprobes(struct list_head *unopt_list, + struct list_head *free_list, + struct list_head *cooling_list) +{ + struct optimized_kprobe *op, *tmp; + lockdep_assert_held(&text_mutex); /* * The optimization/unoptimization refers 'online_cpus' via @@ -553,31 +626,11 @@ static void do_optimize_kprobes(void) */ lockdep_assert_cpus_held(); - /* Optimization never be done when disarmed */ - if (kprobes_all_disarmed || !kprobes_allow_optimization || - list_empty(&optimizing_list)) - return; + if (!list_empty(unopt_list)) + arch_unoptimize_kprobes(unopt_list, free_list); - arch_optimize_kprobes(&optimizing_list); -} - -/* - * Unoptimize (replace a jump with a breakpoint and remove the breakpoint - * if need) kprobes listed on 'unoptimizing_list'. - */ -static void do_unoptimize_kprobes(void) -{ - struct optimized_kprobe *op, *tmp; - - lockdep_assert_held(&text_mutex); - /* See comment in do_optimize_kprobes() */ - lockdep_assert_cpus_held(); - - if (!list_empty(&unoptimizing_list)) - arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); - - /* Loop on 'freeing_list' for disarming and removing from kprobe hash list */ - list_for_each_entry_safe(op, tmp, &freeing_list, list) { + /* Loop on 'free_list' for disarming and removing from kprobe hash list */ + list_for_each_entry_safe(op, tmp, free_list, list) { /* Switching from detour code to origin */ op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; /* Disarm probes if marked disabled and not gone */ @@ -589,18 +642,26 @@ static void do_unoptimize_kprobes(void) * for synchronization, these probes are reclaimed. * (reclaiming is done by do_free_cleaned_kprobes().) */ - hlist_del_rcu(&op->kp.hlist); - } else - list_del_init(&op->list); + hlist_del_init_rcu(&op->kp.hlist); + } else { + /* + * Keep on cooling_list until the quiescence period + * completes so that kprobe_disarmed() remains false and + * unregister_kprobes() does not prematurely free it. + */ + list_move(&op->list, cooling_list); + } } } -/* Reclaim all kprobes on the 'freeing_list' */ -static void do_free_cleaned_kprobes(void) +/* Reclaim all kprobes on the 'free_list' */ +static void do_free_cleaned_kprobes(struct list_head *free_list) { struct optimized_kprobe *op, *tmp; - list_for_each_entry_safe(op, tmp, &freeing_list, list) { + list_for_each_entry_safe(op, tmp, free_list, list) { + struct kprobe *_p; + list_del_init(&op->list); if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) { /* @@ -612,11 +673,10 @@ static void do_free_cleaned_kprobes(void) /* * The aggregator was holding back another probe while it sat on the - * unoptimizing/freeing lists. Now that the aggregator has been fully + * unoptimizing/freeing lists. Now that the aggregator has been fully * reverted we can safely retry the optimization of that sibling. */ - - struct kprobe *_p = get_optimized_kprobe(op->kp.addr); + _p = get_optimized_kprobe(op->kp.addr); if (unlikely(_p)) optimize_kprobe(_p); @@ -626,67 +686,146 @@ static void do_free_cleaned_kprobes(void) static void kick_kprobe_optimizer(void); -/* Kprobe jump optimizer */ -static void kprobe_optimizer(void) +static void optprobe_generation_rcu_cb(struct rcu_head *rcu) { - guard(mutex)(&kprobe_mutex); + struct optprobe_generation *gen; + + gen = container_of(rcu, struct optprobe_generation, rcu); + WRITE_ONCE(gen->ready, true); + wake_up(&kprobe_optimizer_wait); +} + +static void optprobe_finalize_generation(struct optprobe_generation *gen) +{ + struct optimized_kprobe *op, *tmp; + + lockdep_assert_held(&kprobe_mutex); + + scoped_guard(cpus_read_lock) { + guard(mutex)(&text_mutex); + + /* Optimization never be done when disarmed */ + if (!kprobes_all_disarmed && kprobes_allow_optimization && + !list_empty(&gen->optimizing_list)) + arch_optimize_kprobes(&gen->optimizing_list); + } + + /* Free cleaned kprobes after quiescence period */ + do_free_cleaned_kprobes(&gen->freeing_list); + + /* Finalize unoptimized kprobes whose quiescence period completed */ + list_for_each_entry_safe(op, tmp, &gen->cooling_list, list) { + if (kprobe_unused(&op->kp)) { + /* + * Unregistered while quiescence period was in flight. + * Unhash it now and move to cur_gen's freeing list so + * it will be reclaimed after the next quiescence period. + */ + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, &optprobe_gens[optprobe_cur_gen].freeing_list); + kick_kprobe_optimizer(); + } else { + /* Still in use; now safely disarmed */ + list_del_init(&op->list); + if (!kprobe_disabled(&op->kp)) + optimize_kprobe(&op->kp); + } + } + + gen->in_flight = false; + WRITE_ONCE(gen->ready, false); +} + +static bool optprobe_dispatch_generation(void) +{ + struct optprobe_generation *gen; + + lockdep_assert_held(&kprobe_mutex); + + if (!optprobe_can_fire() || !optprobe_has_queued_probes()) + return false; + + gen = &optprobe_gens[optprobe_cur_gen]; scoped_guard(cpus_read_lock) { guard(mutex)(&text_mutex); /* - * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) - * kprobes before waiting for quiesence period. + * Unoptimize kprobes and collect cleaned (unused and disarmed) + * kprobes before waiting for quiescence period. */ - do_unoptimize_kprobes(); - - /* - * Step 2: Wait for quiesence period to ensure all potentially - * preempted tasks to have normally scheduled. Because optprobe - * may modify multiple instructions, there is a chance that Nth - * instruction is preempted. In that case, such tasks can return - * to 2nd-Nth byte of jump instruction. This wait is for avoiding it. - * Note that on non-preemptive kernel, this is transparently converted - * to synchronoze_sched() to wait for all interrupts to have completed. - */ - synchronize_rcu_tasks(); - - /* Step 3: Optimize kprobes after quiesence period */ - do_optimize_kprobes(); - - /* Step 4: Free cleaned kprobes after quiesence period */ - do_free_cleaned_kprobes(); + do_unoptimize_kprobes(&gen->unoptimizing_list, &gen->freeing_list, + &gen->cooling_list); } - /* Step 5: Wake up flushers, and kick optimizer again if needed. */ - optimizer_passes++; - wake_up_var_locked(&optimizer_passes, &kprobe_mutex); + /* Advance cur_gen to the next generation slot */ + optprobe_cur_gen = (optprobe_cur_gen + 1) % OPTPROBE_GEN_MAX; - if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) - kick_kprobe_optimizer(); /*normal kick*/ + gen->in_flight = true; + WRITE_ONCE(gen->ready, false); + + call_rcu_tasks(&gen->rcu, optprobe_generation_rcu_cb); + return true; +} + +/* Kprobe jump optimizer */ +static void kprobe_optimizer(void) +{ + bool progress = false; + int i; + + guard(mutex)(&kprobe_mutex); + + /* Step 1: Finalize any generation whose Tasks RCU grace period completed */ + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + if (READ_ONCE(optprobe_gens[i].ready)) { + optprobe_finalize_generation(&optprobe_gens[i]); + progress = true; + } + } + + /* Step 2: Dispatch waiting room generation if allowed */ + if (optprobe_dispatch_generation()) + progress = true; + + /* Step 3: Wake up flushers if progress was made */ + if (progress) { + optimizer_passes++; + wake_up_var_locked(&optimizer_passes, &kprobe_mutex); + } + + if (optprobe_has_queued_probes() && optprobe_can_fire()) { + /* Probes remain and can be fired immediately (e.g. retried siblings) */ + kick_kprobe_optimizer(); + } } static int kprobe_optimizer_thread(void *data) { while (!kthread_should_stop()) { - /* To avoid hung_task, wait in interruptible state. */ + /* Wait until there is work to do or a generation is ready */ wait_event_interruptible(kprobe_optimizer_wait, - atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE || - kthread_should_stop()); + atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE || + optprobe_has_ready_gens() || + kthread_should_stop()); if (kthread_should_stop()) break; /* - * If it was a normal kick, wait for OPTIMIZE_DELAY. - * This wait can be interrupted by a flush request. + * If it was a normal kick and no generation is ready to finalize, + * wait for OPTIMIZE_DELAY to batch incoming requests. + * This wait can be interrupted by a flush request or a ready generation. */ - if (atomic_read(&optimizer_state) == 1) + if (atomic_read(&optimizer_state) == OPTIMIZER_ST_KICKED && + !optprobe_has_ready_gens()) { wait_event_interruptible_timeout( kprobe_optimizer_wait, atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING || + optprobe_has_ready_gens() || kthread_should_stop(), OPTIMIZE_DELAY); + } if (kthread_should_stop()) break; @@ -711,20 +850,20 @@ static void wait_for_kprobe_optimizer_locked(void) { lockdep_assert_held(&kprobe_mutex); - while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { + while (optprobe_optimizer_busy()) { unsigned long passes = optimizer_passes; - /* - * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's - * idle. If it's already kicked, it will see the state change. - */ - if (atomic_xchg_acquire(&optimizer_state, - OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) - wake_up(&kprobe_optimizer_wait); + /* Wake up optimizer thread if it can make progress */ + if ((optprobe_can_fire() && optprobe_has_queued_probes()) || + optprobe_has_ready_gens()) { + if (atomic_xchg_acquire(&optimizer_state, + OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) + wake_up(&kprobe_optimizer_wait); + } /* * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which - * this drops while sleeping, so a new count means a full pass ran. + * this drops while sleeping, so a new count means progress was made. */ wait_var_event_mutex(&optimizer_passes, optimizer_passes != passes, &kprobe_mutex); @@ -742,10 +881,43 @@ void wait_for_kprobe_optimizer(void) bool optprobe_queued_unopt(struct optimized_kprobe *op) { struct optimized_kprobe *_op; + int i; - list_for_each_entry(_op, &unoptimizing_list, list) { - if (op == _op) - return true; + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + list_for_each_entry(_op, &optprobe_gens[i].unoptimizing_list, list) { + if (op == _op) + return true; + } + } + + return false; +} + +static bool optprobe_queued_freeing(struct optimized_kprobe *op) +{ + struct optimized_kprobe *_op; + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + list_for_each_entry(_op, &optprobe_gens[i].freeing_list, list) { + if (op == _op) + return true; + } + } + + return false; +} + +static bool optprobe_queued_cooling(struct optimized_kprobe *op) +{ + struct optimized_kprobe *_op; + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + list_for_each_entry(_op, &optprobe_gens[i].cooling_list, list) { + if (op == _op) + return true; + } } return false; @@ -779,6 +951,15 @@ static void optimize_kprobe(struct kprobe *p) } return; } + + if (optprobe_queued_cooling(op)) { + /* Under in-flight unoptimization. It will be re-optimized upon finalize */ + return; + } + + if (optprobe_queued_freeing(op)) + list_del_init(&op->list); + op->kp.flags |= KPROBE_FLAG_OPTIMIZED; /* @@ -788,7 +969,7 @@ static void optimize_kprobe(struct kprobe *p) if (WARN_ON_ONCE(!list_empty(&op->list))) return; - list_add(&op->list, &optimizing_list); + list_add(&op->list, &optprobe_gens[optprobe_cur_gen].optimizing_list); kick_kprobe_optimizer(); } @@ -803,14 +984,14 @@ static void force_unoptimize_kprobe(struct optimized_kprobe *op) /* Unoptimize a kprobe if p is optimized */ static void unoptimize_kprobe(struct kprobe *p, bool force) { + struct optprobe_generation *cur; struct optimized_kprobe *op; if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) return; /* This is not an optprobe nor optimized */ op = container_of(p, struct optimized_kprobe, kp); - if (!kprobe_optimized(p)) - return; + cur = &optprobe_gens[optprobe_cur_gen]; if (!list_empty(&op->list)) { if (optprobe_queued_unopt(op)) { @@ -818,10 +999,30 @@ static void unoptimize_kprobe(struct kprobe *p, bool force) if (force) { /* * Forcibly unoptimize the kprobe here, and queue it - * in the freeing list for release afterwards. + * for cooling down (or freeing if unused). */ force_unoptimize_kprobe(op); - list_move(&op->list, &freeing_list); + if (kprobe_unused(&op->kp)) { + if (!kprobe_gone(&op->kp)) + arch_disarm_kprobe(&op->kp); + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, &cur->freeing_list); + } else { + list_move(&op->list, &cur->cooling_list); + } + kick_kprobe_optimizer(); + } + } else if (optprobe_queued_cooling(op)) { + if (force && kprobe_unused(&op->kp)) { + /* + * Already unoptimized, move to freeing list for + * release afterwards. + */ + if (!kprobe_gone(&op->kp)) + arch_disarm_kprobe(&op->kp); + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, &cur->freeing_list); + kick_kprobe_optimizer(); } } else { /* Dequeue from the optimizing queue */ @@ -831,12 +1032,15 @@ static void unoptimize_kprobe(struct kprobe *p, bool force) return; } + if (!kprobe_optimized(p)) + return; + /* Optimized kprobe case */ if (force) { /* Forcibly update the code: this is a special case */ force_unoptimize_kprobe(op); } else { - list_add(&op->list, &unoptimizing_list); + list_add(&op->list, &cur->unoptimizing_list); kick_kprobe_optimizer(); } } @@ -868,23 +1072,29 @@ static void kill_optimized_kprobe(struct kprobe *p) struct optimized_kprobe *op; op = container_of(p, struct optimized_kprobe, kp); - if (!list_empty(&op->list)) - /* Dequeue from the (un)optimization queue */ - list_del_init(&op->list); + if (!list_empty(&op->list)) { + if (kprobe_unused(p)) { + if (optprobe_queued_unopt(op) || optprobe_queued_cooling(op)) { + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, + &optprobe_gens[optprobe_cur_gen].freeing_list); + kick_kprobe_optimizer(); + } else if (!optprobe_queued_freeing(op)) { + list_del_init(&op->list); + } + } else { + list_del_init(&op->list); + } + } op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; - if (kprobe_unused(p)) { - /* - * Unused kprobe is on unoptimizing or freeing list. We move it - * to freeing_list and let the kprobe_optimizer() remove it from - * the kprobe hash list and free it. - */ - if (optprobe_queued_unopt(op)) - list_move(&op->list, &freeing_list); - } - - /* Don't touch the code, because it is already freed. */ - arch_remove_optimized_kprobe(op); + /* + * Don't remove the slot if it is queued for freeing or unoptimization; + * the optimizer will reclaim it after the quiescence period. + */ + if (!optprobe_queued_freeing(op) && !optprobe_queued_unopt(op) && + !optprobe_queued_cooling(op)) + arch_remove_optimized_kprobe(op); } static inline @@ -1060,6 +1270,19 @@ static void __arm_kprobe(struct kprobe *p) optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ } +/* Return true if the probe is queued on unoptimizing lists */ +static bool kprobe_queued_unopt(struct kprobe *p) +{ + struct optimized_kprobe *op; + + if (kprobe_aggrprobe(p)) { + op = container_of(p, struct optimized_kprobe, kp); + if (optprobe_queued_unopt(op)) + return true; + } + return false; +} + /* Remove the breakpoint of a probe. */ static void __disarm_kprobe(struct kprobe *p, bool reopt) { @@ -1070,7 +1293,7 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt) /* Try to unoptimize */ unoptimize_kprobe(p, kprobes_all_disarmed); - if (!kprobe_queued(p)) { + if (!kprobe_queued_unopt(p)) { arch_disarm_kprobe(p); /* If another kprobe was blocked, re-optimize it. */ _p = get_optimized_kprobe(p->addr); @@ -1081,11 +1304,23 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt) static void __init init_optprobe(void) { + int i; + #ifdef __ARCH_WANT_KPROBES_INSN_SLOT /* Init 'kprobe_optinsn_slots' for allocation */ kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; #endif + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + INIT_LIST_HEAD(&optprobe_gens[i].optimizing_list); + INIT_LIST_HEAD(&optprobe_gens[i].unoptimizing_list); + INIT_LIST_HEAD(&optprobe_gens[i].cooling_list); + INIT_LIST_HEAD(&optprobe_gens[i].freeing_list); + optprobe_gens[i].in_flight = false; + optprobe_gens[i].ready = false; + } + optprobe_cur_gen = 0; + init_waitqueue_head(&kprobe_optimizer_wait); atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE); kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL, @@ -3027,47 +3262,22 @@ static int disarm_all_kprobes(void) return ret; } -/* - * XXX: The debugfs bool file interface doesn't allow for callbacks - * when the bool state is switched. We can reuse that facility when - * available - */ -static ssize_t read_enabled_file_bool(struct file *file, - char __user *user_buf, size_t count, loff_t *ppos) +static int kprobes_enabled_set(void *data, u64 val) { - char buf[3]; + if (val) + return arm_all_kprobes(); - if (!kprobes_all_disarmed) - buf[0] = '1'; - else - buf[0] = '0'; - buf[1] = '\n'; - buf[2] = 0x00; - return simple_read_from_buffer(user_buf, count, ppos, buf, 2); + return disarm_all_kprobes(); } -static ssize_t write_enabled_file_bool(struct file *file, - const char __user *user_buf, size_t count, loff_t *ppos) +static int kprobes_enabled_get(void *data, u64 *val) { - bool enable; - int ret; - - ret = kstrtobool_from_user(user_buf, count, &enable); - if (ret) - return ret; - - ret = enable ? arm_all_kprobes() : disarm_all_kprobes(); - if (ret) - return ret; - - return count; + *val = !kprobes_all_disarmed; + return 0; } -static const struct file_operations fops_kp = { - .read = read_enabled_file_bool, - .write = write_enabled_file_bool, - .llseek = default_llseek, -}; +DEFINE_DEBUGFS_ATTRIBUTE(fops_kp, kprobes_enabled_get, + kprobes_enabled_set, "%llu\n"); static int __init debugfs_kprobe_init(void) {
diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index a240d11..517fe42 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c
@@ -646,12 +646,15 @@ static const struct kobj_type klp_ktype_patch = { static void klp_kobj_release_object(struct kobject *kobj) { + struct klp_patch *patch; struct klp_object *obj; obj = container_of(kobj, struct klp_object, kobj); + patch = obj->patch; if (obj->dynamic) klp_free_object_dynamic(obj); + kobject_put(&patch->kobj); } static const struct kobj_type klp_ktype_object = { @@ -662,12 +665,15 @@ static const struct kobj_type klp_ktype_object = { static void klp_kobj_release_func(struct kobject *kobj) { + struct klp_object *obj; struct klp_func *func; func = container_of(kobj, struct klp_func, kobj); + obj = func->obj; if (func->nop) klp_free_func_nop(func); + kobject_put(&obj->kobj); } static const struct kobj_type klp_ktype_func = { @@ -943,7 +949,9 @@ static void klp_init_func_early(struct klp_object *obj, struct klp_func *func) { kobject_init(&func->kobj, &klp_ktype_func); + kobject_get(&obj->kobj); list_add_tail(&func->node, &obj->func_list); + func->obj = obj; } static void klp_init_object_early(struct klp_patch *patch, @@ -951,7 +959,9 @@ static void klp_init_object_early(struct klp_patch *patch, { INIT_LIST_HEAD(&obj->func_list); kobject_init(&obj->kobj, &klp_ktype_object); + kobject_get(&patch->kobj); list_add_tail(&obj->node, &patch->obj_list); + obj->patch = patch; } static void klp_init_patch_early(struct klp_patch *patch)
diff --git a/kernel/module/main.c b/kernel/module/main.c index 4752777..34e0204 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c
@@ -2735,6 +2735,10 @@ static int find_module_sections(struct module *mod, struct load_info *info) mod->btf_base_data = any_section_objs(info, ".BTF.base", 1, &mod->btf_base_data_size); #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = any_section_objs(info, ".BTF.inline", 1, + &mod->btf_inline_data_size); +#endif #ifdef CONFIG_JUMP_LABEL mod->jump_entries = section_objs(info, "__jump_table", sizeof(*mod->jump_entries), @@ -3189,6 +3193,9 @@ static noinline int do_init_module(struct module *mod) mod->btf_data = NULL; mod->btf_base_data = NULL; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = NULL; +#endif /* * We want to free module_init, but be aware that kallsyms may be * walking this within an RCU read section. In all the failure paths, we
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 0ab5916..d8faeb6 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig
@@ -770,7 +770,7 @@ config PROBE_EVENTS_BTF_ARGS depends on HAVE_FUNCTION_ARG_ACCESS_API depends on FPROBE_EVENTS || KPROBE_EVENTS - depends on DEBUG_INFO_BTF && BPF_SYSCALL + depends on DEBUG_INFO_BTF bool "Support BTF function arguments for probe events" default y help
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 3860bc5..1f925cd 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile
@@ -119,7 +119,7 @@ endif obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o -obj-$(CONFIG_PROBE_EVENTS_BTF_ARGS) += trace_btf.o +obj-$(CONFIG_DEBUG_INFO_BTF) += trace_btf.o obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o obj-$(CONFIG_FTRACE_RECORD_RECURSION) += trace_recursion_record.o
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 88c3b5b..dfac5e1 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c
@@ -823,6 +823,7 @@ const struct bpf_func_proto bpf_task_pt_regs_proto = { struct send_signal_irq_work { struct irq_work irq_work; + /* The claim: NULL marks the slot free for the next caller. */ struct task_struct *task; u32 sig; enum pid_type type; @@ -842,6 +843,8 @@ static void do_bpf_send_signal(struct irq_work *entry) group_send_sig_info(work->sig, siginfo, work->task, work->type); put_task_struct(work->task); + /* Release once the fields are consumed. */ + smp_store_release(&work->task, NULL); } static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value) @@ -885,14 +888,14 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc return -EINVAL; work = this_cpu_ptr(&send_signal_work); - if (irq_work_is_busy(&work->irq_work)) + if (cmpxchg(&work->task, NULL, task)) return -EBUSY; /* Add the current task, which is the target of sending signal, * to the irq_work. The current task may change when queued * irq works get executed. */ - work->task = get_task_struct(task); + get_task_struct(task); work->has_siginfo = siginfo == &info; if (work->has_siginfo) copy_siginfo(&work->info, &info); @@ -995,7 +998,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1052,7 +1055,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_MEM_SIZE, @@ -1565,7 +1568,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -2206,6 +2209,7 @@ int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link * return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link); } +/* On success the caller releases *buf with kfree_const() */ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, u32 *fd_type, const char **buf, u64 *probe_offset, u64 *probe_addr, @@ -2215,6 +2219,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, struct bpf_prog *prog; int flags, err = 0; + /* + * The list of dynamic events is walked, and a probe which has siblings + * can be deleted, with its strings, while its event is in use. + */ + guard(mutex)(&event_mutex); + prog = event->prog; if (!prog) return -ENOENT; @@ -2255,6 +2265,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, #endif } + if (!err && *buf) { + *buf = kstrdup_const(*buf, GFP_KERNEL); + if (!*buf) + err = -ENOMEM; + } + return err; }
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 673a54f..c7d74b3 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c
@@ -428,6 +428,7 @@ struct ftrace_profile { #ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; unsigned long long time_squared; + unsigned long long time_max; #endif }; @@ -530,9 +531,9 @@ static int function_stat_headers(struct seq_file *m) { #ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_puts(m, " Function " - "Hit Time Avg s^2\n" + "Hit Time Avg s^2 max\n" " -------- " - "--- ---- --- ---\n"); + "--- ---- --- --- ---\n"); #else seq_puts(m, " Function Hit\n" " -------- ---\n"); @@ -613,6 +614,8 @@ static int function_stat_show(struct seq_file *m, void *v) trace_print_graph_duration(avg, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(stddev, &s); + trace_seq_puts(&s, " "); + trace_print_graph_duration(rec->time_max, &s); trace_print_seq(m, &s); #endif seq_putc(m, '\n'); @@ -920,6 +923,8 @@ static void profile_graph_return(struct ftrace_graph_ret *trace, if (rec) { rec->time += calltime; rec->time_squared += calltime * calltime; + if (calltime > rec->time_max) + rec->time_max = calltime; } } @@ -4426,9 +4431,9 @@ subsys_initcall(ftrace_check_for_weak_functions); static int print_rec(struct seq_file *m, unsigned long ip) { - unsigned long offset; + unsigned long offset = 0; char str[KSYM_SYMBOL_LEN]; - char *modname; + char *modname = NULL; const char *ret; ret = kallsyms_lookup(ip, NULL, &offset, &modname, str); @@ -7702,6 +7707,11 @@ static int ftrace_process_locs(struct module *mod, } addr = ftrace_call_adjust(addr); + /* The architecture may return 0 to reject a patch site */ + if (!addr) { + skipped++; + continue; + } end_offset = (pg->index+1) * sizeof(pg->records[0]); if (end_offset > PAGE_SIZE << pg->order) { @@ -7715,6 +7725,20 @@ static int ftrace_process_locs(struct module *mod, rec->ip = addr; } + /* + * ftrace_release_mod() finds a module's page groups by their first + * record. If every entry was skipped there is none, so unlink the + * new page groups and free them now. + */ + if (mod && !start_pg->index) { + ftrace_pages->next = NULL; + mutex_unlock(&ftrace_lock); + /* Need to synchronize with ftrace_location_range() */ + synchronize_rcu(); + ftrace_free_pages(start_pg); + return 0; + } + if (pg->next) { pg_unuse = pg->next; pg->next = NULL;
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9c4e408..b52f51d 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c
@@ -642,6 +642,11 @@ static __always_inline unsigned int rb_subbuf_max_data_size(struct trace_buffer return rb_subbuf_capacity(buffer) - RB_EVNT_HDR_SIZE - sizeof(event->array[0]); } +static unsigned long rb_calculate_nr_pages(struct trace_buffer *buffer, unsigned long size) +{ + return max(DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)), RING_BUFFER_MIN_NR_PAGES); +} + /** * rb_subbuf_start - Get the start address of a subbuffer * @buffer: A trace buffer @@ -2868,7 +2873,7 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, goto fail_free_buffers; /* Need at least two pages plus the reader page */ - if (nr_pages < 3) + if (nr_pages < RING_BUFFER_MIN_NR_PAGES + 1) goto fail_free_buffers; again: @@ -2880,7 +2885,7 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, ptr += subbuf_size * nr_pages; } if (ptr > end) { - if (nr_pages <= 3) + if (nr_pages <= RING_BUFFER_MIN_NR_PAGES + 1) goto fail_free_buffers; nr_pages--; goto again; @@ -2903,13 +2908,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, if (nr_pages > rb_static_max_pages()) goto fail_free_buffers; - if (nr_pages < 2) + if (nr_pages < RING_BUFFER_MIN_NR_PAGES) goto fail_free_buffers; } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); - if (nr_pages < 2) - nr_pages = 2; + nr_pages = rb_calculate_nr_pages(buffer, size); } cpumask_set_cpu(cpu, buffer->cpumask); @@ -3284,6 +3286,21 @@ static void update_pages_handler(struct work_struct *work) complete(&cpu_buffer->update_done); } +static bool rb_resize_disabled(struct trace_buffer *buffer, int cpu) +{ + lockdep_assert_held(&buffer->mutex); + + if (cpu != RING_BUFFER_ALL_CPUS) + return atomic_read(&buffer->buffers[cpu]->resize_disabled); + + for_each_buffer_cpu(buffer, cpu) { + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return true; + } + + return false; +} + /** * ring_buffer_resize - resize the ring buffer * @buffer: the buffer to resize. @@ -3322,26 +3339,19 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); - nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + nr_pages = rb_calculate_nr_pages(buffer, size); - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; + /* + * Don't succeed if resizing is disabled, as a reader might be + * manipulating the ring buffer and is expecting a sane state while + * this is true. + */ + if (rb_resize_disabled(buffer, cpu_id)) { + err = -EBUSY; + goto out_err_unlock; + } if (cpu_id == RING_BUFFER_ALL_CPUS) { - /* - * Don't succeed if resizing is disabled, as a reader might be - * manipulating the ring buffer and is expecting a sane state while - * this is true. - */ - for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto out_err_unlock; - } - } - /* calculate the pages to update */ for_each_buffer_cpu(buffer, cpu) { cpu_buffer = buffer->buffers[cpu]; @@ -3413,16 +3423,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, if (nr_pages == cpu_buffer->nr_pages) goto out; - /* - * Don't succeed if resizing is disabled, as a reader might be - * manipulating the ring buffer and is expecting a sane state while - * this is true. - */ - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto out_err_unlock; - } - cpu_buffer->nr_pages_to_update = nr_pages - cpu_buffer->nr_pages; @@ -7466,6 +7466,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; + guard(cpus_read_lock)(); + /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); @@ -7475,13 +7477,9 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) old_capacity = rb_subbuf_capacity(buffer); - /* The mmap fast path reads subbuf_order without buffer->mutex. */ - for_each_buffer_cpu(buffer, cpu) { - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) - return -EBUSY; - } + /* Check it is resizable before we touch subbuf_order */ + if (rb_resize_disabled(buffer, RING_BUFFER_ALL_CPUS)) + return -EBUSY; atomic_inc(&buffer->record_disabled); @@ -7492,24 +7490,11 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { - - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - cpu_buffer = buffer->buffers[cpu]; - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto error; - } - /* Update the number of pages to match the new size */ - nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; + nr_pages = rb_calculate_nr_pages(buffer, + old_capacity * buffer->buffers[cpu]->nr_pages); cpu_buffer->nr_pages_to_update = nr_pages; @@ -7531,9 +7516,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) struct list_head old_pages; unsigned long flags; - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - cpu_buffer = buffer->buffers[cpu]; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); @@ -8100,7 +8082,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) } /* allocate minimum pages, user can later expand it */ if (!nr_pages_same) - nr_pages = 2; + nr_pages = RING_BUFFER_MIN_NR_PAGES; buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); if (!buffer->buffers[cpu]) {
diff --git a/kernel/trace/rv/Kconfig b/kernel/trace/rv/Kconfig index efa930f..9bfd429 100644 --- a/kernel/trace/rv/Kconfig +++ b/kernel/trace/rv/Kconfig
@@ -113,6 +113,18 @@ Enables the panic reactor. The panic reactor emits a printk() message if an exception is found and panic()s the system. +config RV_REACTORS_KUNIT + tristate "KUnit tests for RV reactors" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on RV_REACTORS + default KUNIT_ALL_TESTS + help + Enable KUnit tests for RV reactor registration and dispatch. + These tests verify the register/unregister lifecycle, duplicate + rejection, and that rv_react() correctly invokes callbacks. + + If unsure, say N. + config RV_MONITORS_KUNIT_TEST tristate "KUnit tests for RV monitors" if !KUNIT_ALL_TESTS depends on KUNIT && RV && RV_REACTORS
diff --git a/kernel/trace/rv/Makefile b/kernel/trace/rv/Makefile index cdbf68c..c895d81 100644 --- a/kernel/trace/rv/Makefile +++ b/kernel/trace/rv/Makefile
@@ -25,4 +25,5 @@ obj-$(CONFIG_RV_REACTORS) += rv_reactors.o obj-$(CONFIG_RV_REACT_PRINTK) += reactor_printk.o obj-$(CONFIG_RV_REACT_PANIC) += reactor_panic.o +obj-$(CONFIG_RV_REACTORS_KUNIT) += rv_reactors_kunit.o obj-$(CONFIG_RV_MONITORS_KUNIT_TEST) += rv_monitors_test.o
diff --git a/kernel/trace/rv/monitors/wakeup/wakeup.c b/kernel/trace/rv/monitors/wakeup/wakeup.c index 01b4741..2e803e7 100644 --- a/kernel/trace/rv/monitors/wakeup/wakeup.c +++ b/kernel/trace/rv/monitors/wakeup/wakeup.c
@@ -128,7 +128,7 @@ static void disable_wakeup(void) ltl_monitor_destroy(); } -static struct rv_monitor rv_wakeup = { +static struct rv_monitor rv_this = { .name = "wakeup", .description = "Monitor that real-time tasks are not woken by lower-priority tasks", .enable = enable_wakeup, @@ -137,12 +137,12 @@ static struct rv_monitor rv_wakeup = { static int __init register_wakeup(void) { - return rv_register_monitor(&rv_wakeup, &rv_rtapp); + return rv_register_monitor(&rv_this, &rv_rtapp); } static void __exit unregister_wakeup(void) { - rv_unregister_monitor(&rv_wakeup); + rv_unregister_monitor(&rv_this); } module_init(register_wakeup);
diff --git a/kernel/trace/rv/reactor_panic.c b/kernel/trace/rv/reactor_panic.c index 76537b8..db7116c 100644 --- a/kernel/trace/rv/reactor_panic.c +++ b/kernel/trace/rv/reactor_panic.c
@@ -26,8 +26,7 @@ static struct rv_reactor rv_panic = { static int __init register_react_panic(void) { - rv_register_reactor(&rv_panic); - return 0; + return rv_register_reactor(&rv_panic); } static void __exit unregister_react_panic(void)
diff --git a/kernel/trace/rv/reactor_printk.c b/kernel/trace/rv/reactor_printk.c index 48c934e..002a10f 100644 --- a/kernel/trace/rv/reactor_printk.c +++ b/kernel/trace/rv/reactor_printk.c
@@ -25,8 +25,7 @@ static struct rv_reactor rv_printk = { static int __init register_react_printk(void) { - rv_register_reactor(&rv_printk); - return 0; + return rv_register_reactor(&rv_printk); } static void __exit unregister_react_printk(void)
diff --git a/kernel/trace/rv/rv_reactors.c b/kernel/trace/rv/rv_reactors.c index 2f5fc8d..8e57a95 100644 --- a/kernel/trace/rv/rv_reactors.c +++ b/kernel/trace/rv/rv_reactors.c
@@ -314,6 +314,7 @@ int rv_register_reactor(struct rv_reactor *reactor) guard(mutex)(&rv_interface_lock); return __rv_register_reactor(reactor); } +EXPORT_SYMBOL_GPL(rv_register_reactor); /** * rv_unregister_reactor - unregister a rv reactor. @@ -327,6 +328,7 @@ int rv_unregister_reactor(struct rv_reactor *reactor) list_del(&reactor->list); return 0; } +EXPORT_SYMBOL_GPL(rv_unregister_reactor); /* * reacting_on interface. @@ -465,7 +467,15 @@ int init_rv_reactors(struct dentry *root_dir) void rv_react(struct rv_monitor *monitor, const char *msg, ...) { - static DEFINE_WAIT_OVERRIDE_MAP(rv_react_map, LD_WAIT_FREE); + /* + * Reactors must not explicitly take locks, so they should be + * LD_WAIT_FREE. However, reactor callbacks can run with preemption + * enabled, meaning the preempting code (e.g. the scheduler taking + * rq->__lock at LD_WAIT_SPIN) may violate that constraint. Use + * LD_WAIT_SPIN to avoid false-positive lockdep reports. + * But you should still NOT be using locks in reactors. + */ + static DEFINE_WAIT_OVERRIDE_MAP(rv_react_map, LD_WAIT_SPIN); va_list args; if (!rv_reacting_on() || !monitor->react)
diff --git a/kernel/trace/rv/rv_reactors_kunit.c b/kernel/trace/rv/rv_reactors_kunit.c new file mode 100644 index 0000000..3850c6d --- /dev/null +++ b/kernel/trace/rv/rv_reactors_kunit.c
@@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for RV reactor registration and dispatch. + * + * The dispatch tests rely on reacting_on being enabled, since rv_react() + * returns early when it is off. It is on by default when the suites run + * built-in; as a module, re-enable it if disabled via + * /sys/kernel/tracing/rv/reacting_on. + */ + +#include <kunit/test.h> +#include <linux/build_bug.h> +#include <linux/rv.h> +#include <linux/delay.h> +#include "rv.h" + +static struct rv_reactor test_reactor = { + .name = "kunit_test_reactor", + .description = "KUnit test reactor", +}; + +static void reactor_teardown(void *arg) +{ + rv_unregister_reactor(&test_reactor); +} + +static void register_test_reactor(struct kunit *test) +{ + KUNIT_ASSERT_EQ(test, rv_register_reactor(&test_reactor), 0); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, reactor_teardown, NULL), 0); +} + +static void test_double_register(struct kunit *test) +{ + register_test_reactor(test); + KUNIT_EXPECT_EQ(test, rv_register_reactor(&test_reactor), -EINVAL); +} + +static const char long_reactor_name[] = "kunit_reactor_name_too_long_xxx_"; +static_assert(sizeof(long_reactor_name) - 1 >= MAX_RV_REACTOR_NAME_SIZE, + "long_reactor_name must be at least MAX_RV_REACTOR_NAME_SIZE chars"); + +static void test_name_too_long(struct kunit *test) +{ + static struct rv_reactor long_reactor = { + .name = long_reactor_name, + }; + + KUNIT_EXPECT_EQ(test, rv_register_reactor(&long_reactor), -EINVAL); +} + +static struct kunit_case rv_reactor_registration_cases[] = { + KUNIT_CASE(test_double_register), + KUNIT_CASE(test_name_too_long), + {} +}; + +static struct kunit_suite rv_reactor_registration_suite = { + .name = "rv_reactor_registration", + .test_cases = rv_reactor_registration_cases, +}; + +static int react_call_count; + +__printf(1, 0) static void mock_react(const char *msg, va_list args) +{ + react_call_count++; + /* Busy-wait so a timer interrupt fires inside rv_react(). */ + mdelay(20); +} + +static void test_react_no_callback(struct kunit *test) +{ + struct rv_monitor monitor = { + .name = "kunit_null_react", + }; + + react_call_count = 0; + rv_react(&monitor, "no callback"); + + KUNIT_EXPECT_EQ(test, react_call_count, 0); +} + +static void test_react_callback_invoked(struct kunit *test) +{ + struct rv_monitor monitor = { + .name = "kunit_dispatch_monitor", + .react = mock_react, + }; + + react_call_count = 0; + rv_react(&monitor, "callback invocation test"); + KUNIT_EXPECT_EQ(test, react_call_count, 1); +} + +static struct kunit_case rv_react_dispatch_cases[] = { + KUNIT_CASE(test_react_no_callback), + KUNIT_CASE(test_react_callback_invoked), + {} +}; + +static struct kunit_suite rv_react_dispatch_suite = { + .name = "rv_react_dispatch", + .test_cases = rv_react_dispatch_cases, +}; + +kunit_test_suites(&rv_reactor_registration_suite, &rv_react_dispatch_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit tests for RV reactor registration and dispatch");
diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index 49913bb..abfaf9b 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c
@@ -383,8 +383,8 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, void *page; int i; - /* At least 1 reader page and two pages in the ring-buffer */ - if (desc->nr_page_va < 3) + /* At least RING_BUFFER_MIN_NR_PAGES and 1 reader page */ + if (desc->nr_page_va < RING_BUFFER_MIN_NR_PAGES + 1) return -EINVAL; memset(cpu_buffer, 0, sizeof(*cpu_buffer));
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 3749485a..9044f03 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h
@@ -1006,9 +1006,6 @@ struct ftrace_hash { struct rcu_head rcu; }; -struct ftrace_func_entry * -ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); - static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash) { return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));
diff --git a/kernel/trace/trace_btf.c b/kernel/trace/trace_btf.c index ee7a048..f0e0c04d5 100644 --- a/kernel/trace/trace_btf.c +++ b/kernel/trace/trace_btf.c
@@ -2,6 +2,7 @@ #include <linux/btf.h> #include <linux/kernel.h> #include <linux/slab.h> +#include <linux/kallsyms.h> #include "trace_btf.h" @@ -15,6 +16,9 @@ const struct btf_type *btf_find_func_proto(const char *func_name, struct btf **b const struct btf_type *t; s32 id; + if (unlikely(in_atomic() || irqs_disabled())) + return NULL; + id = bpf_find_btf_id(func_name, BTF_KIND_FUNC, btf_p); if (id < 0) return NULL; @@ -59,19 +63,33 @@ struct btf_anon_stack { u32 offset; }; -/* - * Find a member of data structure/union by name and return it. - * Return NULL if not found, or ERR_PTR(-EINVAL) if parameter is invalid. - * If the member is a member of an anonymous union/structure, the bit offset - * of that anonymous union/structure is stored into @anon_offset. - * If @member_type is non-NULL, the actual containing structure/union type - * of the found member is stored into @member_type. +/** + * btf_find_struct_member() - Find a member of data structure/union by name + * @btf: Pointer to BTF object. + * @type: Pointer to the struct/union BTF type to search within. + * @member_name: Name of the member to find. + * @anon_offset: Pointer to store bit offset of enclosing anonymous union/struct. + * @container_type: Pointer to store the actual containing struct/union type. + * + * Find a member of data structure/union by name, traversing into anonymous + * unions and structures if necessary. + * + * If the member is inside an anonymous union/structure, the bit offset + * of that anonymous union/structure from @type is stored into @anon_offset. + * If @container_type is non-NULL, the actual struct/union BTF type that + * directly contains the found member is stored into @container_type. + * + * Return: + * * Pointer to &struct btf_member on success. + * * %NULL if the member was not found. + * * %ERR_PTR(-EINVAL) if @type is not a struct/union. + * * %ERR_PTR(-ENOMEM) if memory allocation failed. */ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, u32 *anon_offset, - const struct btf_type **member_type) + const struct btf_type **container_type) { struct btf_anon_stack *anon_stack; const struct btf_member *member; @@ -103,8 +121,8 @@ const struct btf_member *btf_find_struct_member(struct btf *btf, if (name && !strcmp(member_name, name)) { if (anon_offset) *anon_offset = cur_offset; - if (member_type) - *member_type = type; + if (container_type) + *container_type = type; goto out; } } @@ -123,3 +141,85 @@ const struct btf_member *btf_find_struct_member(struct btf *btf, return member; } +void btf_trim_retval(unsigned long func, unsigned long *retval, bool *print_retval, + int *fmt, bool hex) +{ + const struct btf_type *t; + char name[KSYM_NAME_LEN]; + struct btf *btf; + u32 v, msb; + bool signed_type; + int kind; + + if (lookup_symbol_name(func, name)) + return; + + t = btf_find_func_proto(name, &btf); + if (IS_ERR_OR_NULL(t)) + return; + + t = btf_type_skip_modifiers(btf, t->type, NULL); + kind = t ? BTF_INFO_KIND(t->info) : BTF_KIND_UNKN; + switch (kind) { + case BTF_KIND_UNKN: + *print_retval = false; + break; + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: + if (kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION) { + *fmt = RETVAL_FMT_HEX; + signed_type = false; + } else { + *fmt = RETVAL_FMT_DEC; + signed_type = btf_type_kflag(t); + if (!signed_type) + *fmt |= RETVAL_FMT_UNSIGNED; + } + + if (t->size > sizeof(unsigned long)) { + *fmt |= RETVAL_FMT_TRUNC; + msb = BITS_PER_LONG - 1; + } else { + msb = min_t(u32, BITS_PER_BYTE * t->size - 1, + BITS_PER_LONG - 1); + *retval &= GENMASK(msb, 0); + } + if (signed_type && t->size && !hex) + *retval = sign_extend64(*retval, msb); + break; + case BTF_KIND_INT: + v = *(u32 *)(t + 1); + signed_type = false; + if (BTF_INT_ENCODING(v) == BTF_INT_BOOL) { + *fmt = RETVAL_FMT_BOOL; + msb = 0; + } else { + signed_type = BTF_INT_ENCODING(v) & BTF_INT_SIGNED; + if (signed_type) + *fmt = RETVAL_FMT_DEC; + else + *fmt = RETVAL_FMT_HEX; + + if (t->size > sizeof(unsigned long)) { + *fmt |= RETVAL_FMT_TRUNC; + msb = BITS_PER_LONG - 1; + } else { + msb = min_t(u32, BTF_INT_BITS(v) - 1, + BITS_PER_LONG - 1); + } + } + *retval &= GENMASK(msb, 0); + if (signed_type && !hex) + *retval = sign_extend64(*retval, msb); + break; + default: + *fmt = RETVAL_FMT_HEX; + break; + } + + if (*print_retval) + *fmt |= RETVAL_FMT_BTF; + btf_put(btf); +}
diff --git a/kernel/trace/trace_btf.h b/kernel/trace/trace_btf.h index 4bd26bc..b3d408e 100644 --- a/kernel/trace/trace_btf.h +++ b/kernel/trace/trace_btf.h
@@ -1,6 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/btf.h> +enum { + RETVAL_FMT_HEX = BIT(0), + RETVAL_FMT_DEC = BIT(1), + RETVAL_FMT_BOOL = BIT(2), + RETVAL_FMT_TRUNC = BIT(3), + RETVAL_FMT_BTF = BIT(4), + RETVAL_FMT_UNSIGNED = BIT(5), +}; + const struct btf_type *btf_find_func_proto(const char *func_name, struct btf **btf_p); const struct btf_param *btf_get_func_param(const struct btf_type *func_proto, @@ -9,4 +18,14 @@ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, u32 *anon_offset, - const struct btf_type **member_type); + const struct btf_type **container_type); + +#ifdef CONFIG_DEBUG_INFO_BTF +void btf_trim_retval(unsigned long func, unsigned long *retval, bool *print_retval, + int *fmt, bool hex); +#else +static inline void btf_trim_retval(unsigned long func, unsigned long *retval, + bool *print_retval, int *fmt, bool hex) +{ +} +#endif
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index c879d43..acaba99 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c
@@ -121,6 +121,7 @@ static ftrace_func_t select_trace_function(u32 flags_val) case TRACE_FUNC_OPT_ARGS: return function_args_trace_call; case TRACE_FUNC_OPT_STACK: + case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_ARGS: return function_stack_trace_call; case TRACE_FUNC_OPT_NO_REPEATS: return function_no_repeats_trace_call; @@ -295,7 +296,9 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, if (likely(disabled == 1)) { trace_ctx = tracing_gen_ctx_flags(flags); - trace_function(tr, ip, parent_ip, trace_ctx, NULL); + if (!(tr->current_trace_flags->val & TRACE_FUNC_OPT_ARGS)) + fregs = NULL; + trace_function(tr, ip, parent_ip, trace_ctx, fregs); #ifdef CONFIG_UNWINDER_FRAME_POINTER if (ftrace_pids_enabled(op)) skip++;
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index c5befd0..78a6083 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c
@@ -15,6 +15,7 @@ #include "trace.h" #include "trace_output.h" +#include "trace_btf.h" /* When set, irq functions might be ignored */ static int ftrace_graph_skip_irqs; @@ -876,9 +877,10 @@ static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entr { unsigned long err_code = 0; unsigned long retval = 0; + bool hex_format; bool print_retaddr = false; bool print_retval = false; - bool hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); + int retval_fmt = 0; #ifdef CONFIG_FUNCTION_GRAPH_RETVAL retval = graph_ret->retval; @@ -889,17 +891,38 @@ static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entr print_retaddr = !!(opt_flags & TRACE_GRAPH_PRINT_RETADDR); #endif - if (print_retval && retval && !hex_format) { - /* Check if the return value matches the negative format */ - if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && - (((u64)retval) >> 32) == 0) { - err_code = sign_extend64(retval, 31); - } else { - err_code = retval; - } + if (print_retval) { + int fmt = RETVAL_FMT_HEX; - if (!IS_ERR_VALUE(err_code)) - err_code = 0; + hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); + btf_trim_retval((unsigned long)func, &retval, &print_retval, &fmt, + hex_format); + if (print_retval) { + if (hex_format) + retval_fmt = RETVAL_FMT_HEX; + + if (retval && retval_fmt != RETVAL_FMT_HEX && + !(fmt & RETVAL_FMT_BTF)) { + /* Check if the return value matches the negative format */ + if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && + (((u64)retval) >> 32) == 0) { + err_code = sign_extend64(retval, 31); + } else { + err_code = retval; + } + + if (!IS_ERR_VALUE(err_code)) + err_code = 0; + } + + if (retval_fmt == RETVAL_FMT_HEX) { + retval_fmt |= (fmt & RETVAL_FMT_TRUNC); + } else { + if (err_code && fmt & RETVAL_FMT_HEX) + fmt = (fmt & ~RETVAL_FMT_HEX) | RETVAL_FMT_DEC; + retval_fmt = fmt; + } + } } if (entry) { @@ -926,10 +949,17 @@ static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entr trace_flags, false); if (print_retval) { - if (hex_format || (err_code == 0)) + if (retval_fmt & RETVAL_FMT_HEX) trace_seq_printf(s, " ret=0x%lx", retval); + else if (retval_fmt & RETVAL_FMT_BOOL) + trace_seq_printf(s, " ret=%s", retval ? "true" : "false"); + else if (retval_fmt & RETVAL_FMT_UNSIGNED) + trace_seq_printf(s, " ret=%lu", retval); else - trace_seq_printf(s, " ret=%ld", err_code); + trace_seq_printf(s, " ret=%ld", err_code ?: retval); + + if (retval_fmt & RETVAL_FMT_TRUNC) + trace_seq_printf(s, "(trunc)"); } if (!entry || print_retval || print_retaddr)
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 88518c5..a880a90 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c
@@ -625,7 +625,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty { const struct btf_type *type = *ptype; const struct btf_member *field; - const struct btf_type *mtype; + const struct btf_type *container_type; struct btf *btf = ctx_btf(ctx); char *fieldname = *pfieldname; int bitoffs = 0; @@ -641,7 +641,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty anon_offs = 0; field = btf_find_struct_member(btf, type, fieldname, - &anon_offs, &mtype); + &anon_offs, &container_type); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return PTR_ERR(field); @@ -654,7 +654,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty bitoffs += anon_offs; /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(mtype)) { + if (btf_type_kflag(container_type)) { bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); } else { @@ -2315,7 +2315,7 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], else *new_argc = argc; - new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL); + new_argv = kcalloc(*new_argc, sizeof(*new_argv), GFP_KERNEL); if (!new_argv) return ERR_PTR(-ENOMEM);
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 004188a..6759087 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug
@@ -425,6 +425,12 @@ otherwise it would emit malformed kernel and module binaries when using DEBUG_INFO_BTF_MODULES. +config PAHOLE_HAS_INLINE + def_bool PAHOLE_VERSION >= 132 + help + Support for the "inline" BTF feature is available. It encodes + information about inline sites and how to retrieve their parameters. + config DEBUG_INFO_BTF_MODULES bool "Generate BTF type information for kernel modules" default y @@ -432,6 +438,18 @@ help Generate compact split BTF type information for kernel modules. +config DEBUG_INFO_BTF_INLINE + tristate "Provide information about inline sites in BTF" + default n + depends on DEBUG_INFO_BTF && PAHOLE_HAS_INLINE && SYSFS + select DEBUG_INFO_BTF_MODULES if MODULES + help + Generate information about inline sites in .BTF.inline sections. + These sections contain split BTF relative to the kernel or module BTF + and are made available in /sys/kernel/btf with a ".inline" suffix. + The information describes inline locations and how to retrieve their + associated parameters. + config MODULE_ALLOW_BTF_MISMATCH bool "Allow loading modules with non-matching BTF type info" depends on DEBUG_INFO_BTF_MODULES @@ -2649,19 +2667,6 @@ If unsure, say N. -config TEST_BPF - tristate "Test BPF filter functionality" - depends on m && NET - help - This builds the "test_bpf" module that runs various test vectors - against the BPF interpreter or BPF JIT compiler depending on the - current setting. This is in particular useful for BPF JIT compiler - development, but also to run regression tests against changes in - the interpreter code. It also enables test stubs for eBPF maps and - verifier used by user space verifier testsuite. - - If unsure, say N. - config FIND_BIT_BENCHMARK tristate "Test find_bit functions" help
diff --git a/lib/Makefile b/lib/Makefile index 3104710..a2afe95 100644 --- a/lib/Makefile +++ b/lib/Makefile
@@ -66,7 +66,6 @@ obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o obj-$(CONFIG_REGION_ALLOC_BENCHMARK) += region_alloc_benchmark.o obj-$(CONFIG_FIND_BIT_BENCHMARK_RUST) += find_bit_benchmark_rust.o -obj-$(CONFIG_TEST_BPF) += test_bpf.o test_dhry-objs := dhry_1.o dhry_2.o dhry_run.o obj-$(CONFIG_TEST_DHRY) += test_dhry.o obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o
diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 89c88e3..819685d 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c
@@ -187,12 +187,9 @@ static inline void * __init xbc_alloc_mem(size_t size) return memblock_alloc(size, SMP_CACHE_BYTES); } -static inline void __init xbc_free_mem(void *addr, size_t size, bool early) +static inline void __init xbc_free_mem(void *addr, size_t size) { - if (early) - memblock_free(addr, size); - else if (addr) - memblock_free(addr, size); + memblock_free(addr, size); } #else /* !__KERNEL__ */ @@ -202,7 +199,7 @@ static inline void *xbc_alloc_mem(size_t size) return calloc(1, size); } -static inline void xbc_free_mem(void *addr, size_t size, bool early) +static inline void xbc_free_mem(void *addr, size_t size) { free(addr); } @@ -539,18 +536,21 @@ static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) /** - * xbc_snprint_cmdline() - Render bootconfig keys under @root as a cmdline string + * xbc_snprint_cmdline_filter() - Render bootconfig keys as cmdline string * @buf: Destination buffer (may be NULL when @size is 0 to query the length) * @size: Size of @buf in bytes * @root: Subtree root whose key=value pairs should be rendered + * @filter: Filter callback (returns true to include, false to skip; may be NULL) + * @data: Private context passed to @filter * * Walk all key/value pairs under @root and emit them as a space-separated * cmdline string into @buf. Values containing whitespace are quoted with - * double quotes. Returns the number of bytes that would be written if @buf - * were large enough (matching snprintf semantics), or a negative errno on - * failure. + * double quotes. Keys for which @filter returns false are omitted. + * Returns the number of bytes that would be written if @buf were large enough + * (matching snprintf semantics), or a negative errno on failure. */ -int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) +int __init xbc_snprint_cmdline_filter(char *buf, size_t size, struct xbc_node *root, + xbc_cmdline_filter_fn filter, void *data) { struct xbc_node *knode, *vnode; const char *val, *q; @@ -582,6 +582,9 @@ int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) if (ret < 0) return ret; + if (filter && !filter(knode, xbc_namebuf, data)) + continue; + vnode = xbc_node_get_child(knode); if (!vnode) { ret = snprintf(buf ? buf + len : NULL, rest(len, size), @@ -836,7 +839,6 @@ static int __init xbc_parse_array(char **__v) return -ENOMEM; *__v = next; } while (c == ','); - node->child = 0; return c; } @@ -1003,9 +1005,30 @@ static int __init xbc_close_brace(char **k, char *n) return __xbc_close_brace(n - 1); } +#ifndef __KERNEL__ +/* Sanity check for regression: node indices must be within bounds */ +static int __init xbc_sanity_check_tree(void) +{ + int i; + + for (i = 0; i < xbc_node_num; i++) { + if (xbc_nodes[i].next >= xbc_node_num) { + return xbc_parse_error("No closing brace", + xbc_node_get_data(xbc_nodes + i)); + } + if (xbc_nodes[i].child >= xbc_node_num) { + return xbc_parse_error("Broken child node", + xbc_node_get_data(xbc_nodes + i)); + } + } + + return 0; +} +#endif + static int __init xbc_verify_tree(void) { - int i, depth; + int depth; size_t len, wlen; struct xbc_node *n, *m; @@ -1022,17 +1045,6 @@ static int __init xbc_verify_tree(void) return -ENOENT; } - for (i = 0; i < xbc_node_num; i++) { - if (xbc_nodes[i].next >= xbc_node_num) { - return xbc_parse_error("No closing brace", - xbc_node_get_data(xbc_nodes + i)); - } - if (xbc_nodes[i].child >= xbc_node_num) { - return xbc_parse_error("Broken child node", - xbc_node_get_data(xbc_nodes + i)); - } - } - /* Key tree limitation check */ n = &xbc_nodes[0]; depth = 1; @@ -1119,24 +1131,30 @@ static int __init xbc_parse_tree(void) } } while (!ret); + if (!ret) { + while (p < xbc_data + xbc_data_size - 1 && *p == '\0') + p++; + if (p < xbc_data + xbc_data_size - 1) + ret = xbc_parse_error("Unexpected data after null character", p); + } + return ret; } /** - * _xbc_exit() - Clean up all parsed bootconfig - * @early: Set true if this is called before budy system is initialized. + * xbc_exit() - Clean up all parsed bootconfig * * This clears all data structures of parsed bootconfig on memory. * If you need to reuse xbc_init() with new boot config, you can * use this. */ -void __init _xbc_exit(bool early) +void __init xbc_exit(void) { - xbc_free_mem(xbc_data, xbc_data_size, early); + xbc_free_mem(xbc_data, xbc_data_size); xbc_data = NULL; xbc_data_size = 0; xbc_node_num = 0; - xbc_free_mem(xbc_nodes, sizeof(struct xbc_node) * XBC_NODE_MAX, early); + xbc_free_mem(xbc_nodes, sizeof(struct xbc_node) * XBC_NODE_MAX); xbc_nodes = NULL; brace_index = 0; } @@ -1189,20 +1207,24 @@ int __init xbc_init(const char *data, size_t size, const char **emsg, int *epos) if (!xbc_nodes) { if (emsg) *emsg = "Failed to allocate bootconfig nodes"; - _xbc_exit(true); + xbc_exit(); return -ENOMEM; } ret = xbc_parse_tree(); if (!ret) ret = xbc_verify_tree(); +#ifndef __KERNEL__ + if (!ret) + ret = xbc_sanity_check_tree(); +#endif if (ret < 0) { if (epos) *epos = xbc_err_pos; if (emsg) *emsg = xbc_err_msg; - _xbc_exit(true); + xbc_exit(); } else { ret = xbc_node_num; }
diff --git a/lib/buildid.c b/lib/buildid.c index c4b7376..995a327 100644 --- a/lib/buildid.c +++ b/lib/buildid.c
@@ -47,6 +47,10 @@ static int freader_get_folio(struct freader *r, loff_t file_off) freader_put_folio(r); + /* the page index must fit in pgoff_t */ + if (file_off < 0 || file_off > MAX_LFS_FILESIZE) + return -EFAULT; + /* only use page cache lookup - fail if not already cached */ r->folio = filemap_get_folio(r->file->f_mapping, file_off >> PAGE_SHIFT);
diff --git a/lib/test_bpf.c b/lib/test_bpf.c deleted file mode 100644 index af6f334..0000000 --- a/lib/test_bpf.c +++ /dev/null
@@ -1,15934 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Testsuite for BPF interpreter and BPF JIT compiler - * - * Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/filter.h> -#include <linux/bpf.h> -#include <linux/skbuff.h> -#include <linux/netdevice.h> -#include <linux/if_vlan.h> -#include <linux/prandom.h> -#include <linux/highmem.h> -#include <linux/sched.h> - -/* General test specific settings */ -#define MAX_SUBTESTS 3 -#define MAX_TESTRUNS 1000 -#define MAX_DATA 128 -#define MAX_INSNS 512 -#define MAX_K 0xffffFFFF - -/* Few constants used to init test 'skb' */ -#define SKB_TYPE 3 -#define SKB_MARK 0x1234aaaa -#define SKB_HASH 0x1234aaab -#define SKB_QUEUE_MAP 123 -#define SKB_VLAN_TCI 0xffff -#define SKB_VLAN_PRESENT 1 -#define SKB_DEV_IFINDEX 577 -#define SKB_DEV_TYPE 588 - -/* Redefine REGs to make tests less verbose */ -#define R0 BPF_REG_0 -#define R1 BPF_REG_1 -#define R2 BPF_REG_2 -#define R3 BPF_REG_3 -#define R4 BPF_REG_4 -#define R5 BPF_REG_5 -#define R6 BPF_REG_6 -#define R7 BPF_REG_7 -#define R8 BPF_REG_8 -#define R9 BPF_REG_9 -#define R10 BPF_REG_10 - -/* Flags that can be passed to test cases */ -#define FLAG_NO_DATA BIT(0) -#define FLAG_EXPECTED_FAIL BIT(1) -#define FLAG_SKB_FRAG BIT(2) -#define FLAG_VERIFIER_ZEXT BIT(3) -#define FLAG_LARGE_MEM BIT(4) - -enum { - CLASSIC = BIT(6), /* Old BPF instructions only. */ - INTERNAL = BIT(7), /* Extended instruction set. */ -}; - -#define TEST_TYPE_MASK (CLASSIC | INTERNAL) - -struct bpf_test { - const char *descr; - union { - struct sock_filter insns[MAX_INSNS]; - struct bpf_insn insns_int[MAX_INSNS]; - struct { - void *insns; - unsigned int len; - } ptr; - } u; - __u8 aux; - __u8 data[MAX_DATA]; - struct { - int data_size; - __u32 result; - } test[MAX_SUBTESTS]; - int (*fill_helper)(struct bpf_test *self); - int expected_errcode; /* used when FLAG_EXPECTED_FAIL is set in the aux */ - __u8 frag_data[MAX_DATA]; - int stack_depth; /* for eBPF only, since tests don't call verifier */ - int nr_testruns; /* Custom run count, defaults to MAX_TESTRUNS if 0 */ -}; - -/* Large test cases need separate allocation and fill handler. */ - -static int bpf_fill_maxinsns1(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - __u32 k = ~0; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++, k--) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, k); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns2(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns3(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - struct rnd_state rnd; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - prandom_seed_state(&rnd, 3141592653589793238ULL); - - for (i = 0; i < len - 1; i++) { - __u32 k = prandom_u32_state(&rnd); - - insn[i] = __BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, k); - } - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns4(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS + 1; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns5(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns6(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_VLAN_TAG_PRESENT); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns7(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 4; i++) - insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_CPU); - - insn[len - 4] = __BPF_STMT(BPF_MISC | BPF_TAX, 0); - insn[len - 3] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_CPU); - insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0); - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns8(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i, jmp_off = len - 3; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_JUMP(BPF_JMP | BPF_JGT, 0xffffffff, jmp_off--, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns9(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = BPF_JMP_IMM(BPF_JA, 0, 0, len - 2); - insn[1] = BPF_ALU32_IMM(BPF_MOV, R0, 0xcbababab); - insn[2] = BPF_EXIT_INSN(); - - for (i = 3; i < len - 2; i++) - insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xfefefefe); - - insn[len - 2] = BPF_EXIT_INSN(); - insn[len - 1] = BPF_JMP_IMM(BPF_JA, 0, 0, -(len - 1)); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns10(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS, hlen = len - 2; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < hlen / 2; i++) - insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 2 - 2 * i); - for (i = hlen - 1; i > hlen / 2; i--) - insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 1 - 2 * i); - - insn[hlen / 2] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen / 2 - 1); - insn[hlen] = BPF_ALU32_IMM(BPF_MOV, R0, 0xabababac); - insn[hlen + 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int __bpf_fill_ja(struct bpf_test *self, unsigned int len, - unsigned int plen) -{ - struct sock_filter *insn; - unsigned int rlen; - int i, j; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - rlen = (len % plen) - 1; - - for (i = 0; i + plen < len; i += plen) - for (j = 0; j < plen; j++) - insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, - plen - 1 - j, 0, 0); - for (j = 0; j < rlen; j++) - insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, rlen - 1 - j, - 0, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xababcbac); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns11(struct bpf_test *self) -{ - /* Hits 70 passes on x86_64 and triggers NOPs padding. */ - return __bpf_fill_ja(self, BPF_MAXINSNS, 68); -} - -static int bpf_fill_maxinsns12(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns13(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 3; i++) - insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0); - - insn[len - 3] = __BPF_STMT(BPF_LD | BPF_IMM, 0xabababab); - insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0); - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_ja(struct bpf_test *self) -{ - /* Hits exactly 11 passes on x86_64 JIT. */ - return __bpf_fill_ja(self, 12, 9); -} - -static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 1; i += 2) { - insn[i] = __BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 0); - insn[i + 1] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU); - } - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xbee); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int __bpf_fill_stxdw(struct bpf_test *self, int size) -{ - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = BPF_ALU32_IMM(BPF_MOV, R0, 1); - insn[1] = BPF_ST_MEM(size, R10, -40, 42); - - for (i = 2; i < len - 2; i++) - insn[i] = BPF_STX_XADD(size, R10, R0, -40); - - insn[len - 2] = BPF_LDX_MEM(size, R0, R10, -40); - insn[len - 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - self->stack_depth = 40; - - return 0; -} - -static int bpf_fill_stxw(struct bpf_test *self) -{ - return __bpf_fill_stxdw(self, BPF_W); -} - -static int bpf_fill_stxdw(struct bpf_test *self) -{ - return __bpf_fill_stxdw(self, BPF_DW); -} - -static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) -{ - struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; - - memcpy(insns, tmp, sizeof(tmp)); - return 2; -} - -/* - * Branch conversion tests. Complex operations can expand to a lot - * of instructions when JITed. This in turn may cause jump offsets - * to overflow the field size of the native instruction, triggering - * a branch conversion mechanism in some JITs. - */ -static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm, bool alu32) -{ - struct bpf_insn *insns; - int len = S16_MAX + 5; - int i; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - i = __bpf_ld_imm64(insns, R1, 0x0123456789abcdefULL); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insns[i++] = BPF_JMP_IMM(jmp, R0, imm, S16_MAX); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 2); - insns[i++] = BPF_EXIT_INSN(); - - while (i < len - 1) { - static const int ops[] = { - BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, - BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, - }; - int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; - - if ((i & 1) || alu32) - insns[i++] = BPF_ALU32_REG(op, R0, R1); - else - insns[i++] = BPF_ALU64_REG(op, R0, R1); - } - - insns[i++] = BPF_EXIT_INSN(); - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* Branch taken by runtime decision */ -static int bpf_fill_max_jmp_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 1, true); -} - -static int bpf_fill_max_jmp_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 1, false); -} - -/* Branch not taken by runtime decision */ -static int bpf_fill_max_jmp_not_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 0, true); -} - -static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 0, false); -} - -/* Branch always taken, known at JIT time */ -static int bpf_fill_max_jmp_always_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JGE, 0, true); -} - -static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JGE, 0, false); -} - -/* Branch never taken, known at JIT time */ -static int bpf_fill_max_jmp_never_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JLT, 0, true); -} - -static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JLT, 0, false); -} - -/* ALU result computation used in tests */ -enum { F_ALU32 = 1, F_SIGNED = 2 }; - -static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op, u32 flags) -{ - bool is_signed = flags & F_SIGNED; - - /* Narrow operands for ALU32 */ - if (flags & F_ALU32) { - if (is_signed) { - v1 = (u64)(s32)v1; - v2 = (u64)(s32)v2; - } else { - v1 = (u32)v1; - v2 = (u32)v2; - } - } - - *res = 0; - switch (op) { - case BPF_MOV: - *res = v2; - break; - case BPF_AND: - *res = v1 & v2; - break; - case BPF_OR: - *res = v1 | v2; - break; - case BPF_XOR: - *res = v1 ^ v2; - break; - case BPF_LSH: - *res = v1 << v2; - break; - case BPF_RSH: - *res = v1 >> v2; - break; - case BPF_ARSH: - *res = v1 >> v2; - if (v2 > 0 && v1 > S64_MAX) - *res |= ~0ULL << (64 - v2); - break; - case BPF_ADD: - *res = v1 + v2; - break; - case BPF_SUB: - *res = v1 - v2; - break; - case BPF_MUL: - *res = v1 * v2; - break; - case BPF_DIV: - if (v2 == 0) - return false; - if (!is_signed) { - *res = div64_u64(v1, v2); - } else { - if ((s64)v2 == -1) /* Handled by verifier */ - return false; - *res = (u64)div64_s64(v1, v2); - } - break; - case BPF_MOD: - if (v2 == 0) - return false; - if (!is_signed) { - div64_u64_rem(v1, v2, res); - } else { - if ((s64)v2 == -1) - return false; - /* - * Avoid s64 % s64 which generates __moddi3 on - * 32-bit architectures. Use div64_s64 instead. - */ - *res = (u64)((s64)v1 - div64_s64(v1, v2) * (s64)v2); - } - break; - } - return true; -} - -/* Test an ALU shift operation for all valid shift values */ -static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, - u8 mode, u32 flags) -{ - static const s64 regs[] = { - 0x0123456789abcdefLL, /* dword > 0, word < 0 */ - 0xfedcba9876543210LL, /* dword < 0, word > 0 */ - 0xfedcba0198765432LL, /* dword < 0, word < 0 */ - 0x0123458967abcdefLL, /* dword > 0, word > 0 */ - }; - int bits = (flags & F_ALU32) ? 32 : 64; - int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; - struct bpf_insn *insn; - int imm, k; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (k = 0; k < ARRAY_SIZE(regs); k++) { - s64 reg = regs[k]; - - i += __bpf_ld_imm64(&insn[i], R3, reg); - - for (imm = 0; imm < bits; imm++) { - u64 val; - - /* Perform operation */ - insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); - if (flags & F_ALU32) { - if (mode == BPF_K) - insn[i++] = BPF_ALU32_IMM(op, R1, imm); - else - insn[i++] = BPF_ALU32_REG(op, R1, R2); - - if (op == BPF_ARSH) - reg = (s32)reg; - else - reg = (u32)reg; - __bpf_alu_result(&val, reg, imm, op, 0); - val = (u32)val; - } else { - if (mode == BPF_K) - insn[i++] = BPF_ALU64_IMM(op, R1, imm); - else - insn[i++] = BPF_ALU64_REG(op, R1, R2); - __bpf_alu_result(&val, reg, imm, op, 0); - } - - /* - * When debugging a JIT that fails this test, one - * can write the immediate value to R0 here to find - * out which operand values that fail. - */ - - /* Load reference and check the result */ - i += __bpf_ld_imm64(&insn[i], R4, val); - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R4, 1); - insn[i++] = BPF_EXIT_INSN(); - } - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, 0); -} - -static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, 0); -} - -static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, 0); -} - -static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, 0); -} - -static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, 0); -} - -static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, 0); -} - -static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, F_ALU32); -} - -static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, F_ALU32); -} - -static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, F_ALU32); -} - -/* - * Test an ALU register shift operation for all valid shift values - * for the case when the source and destination are the same. - */ -static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, - u32 flags) -{ - int bits = (flags & F_ALU32) ? 32 : 64; - int len = 3 + 6 * bits; - struct bpf_insn *insn; - int i = 0; - u64 val; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (val = 0; val < bits; val++) { - u64 res; - - /* Perform operation */ - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); - if (flags & F_ALU32) - insn[i++] = BPF_ALU32_REG(op, R1, R1); - else - insn[i++] = BPF_ALU64_REG(op, R1, R1); - - /* Compute the reference result */ - __bpf_alu_result(&res, val, val, op, 0); - if (flags & F_ALU32) - res = (u32)res; - i += __bpf_ld_imm64(&insn[i], R2, res); - - /* Check the actual result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, 0); -} - -static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, 0); -} - -static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, 0); -} - -static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, F_ALU32); -} - -static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, F_ALU32); -} - -static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, F_ALU32); -} - -/* - * Common operand pattern generator for exhaustive power-of-two magnitudes - * tests. The block size parameters can be adjusted to increase/reduce the - * number of combinatons tested and thereby execution speed and memory - * footprint. - */ - -static inline s64 value(int msb, int delta, int sign) -{ - return sign * (1LL << msb) + delta; -} - -static int __bpf_fill_pattern(struct bpf_test *self, void *arg, - int dbits, int sbits, int block1, int block2, - int (*emit)(struct bpf_test*, void*, - struct bpf_insn*, s64, s64)) -{ - static const int sgn[][2] = {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}; - struct bpf_insn *insns; - int di, si, bt, db, sb; - int count, len, k; - int extra = 1 + 2; - int i = 0; - - /* Total number of iterations for the two pattern */ - count = (dbits - 1) * (sbits - 1) * block1 * block1 * ARRAY_SIZE(sgn); - count += (max(dbits, sbits) - 1) * block2 * block2 * ARRAY_SIZE(sgn); - - /* Compute the maximum number of insns and allocate the buffer */ - len = extra + count * (*emit)(self, arg, NULL, 0, 0); - insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); - if (!insns) - return -ENOMEM; - - /* Add head instruction(s) */ - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - /* - * Pattern 1: all combinations of power-of-two magnitudes and sign, - * and with a block of contiguous values around each magnitude. - */ - for (di = 0; di < dbits - 1; di++) /* Dst magnitudes */ - for (si = 0; si < sbits - 1; si++) /* Src magnitudes */ - for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ - for (db = -(block1 / 2); - db < (block1 + 1) / 2; db++) - for (sb = -(block1 / 2); - sb < (block1 + 1) / 2; sb++) { - s64 dst, src; - - dst = value(di, db, sgn[k][0]); - src = value(si, sb, sgn[k][1]); - i += (*emit)(self, arg, - &insns[i], - dst, src); - } - /* - * Pattern 2: all combinations for a larger block of values - * for each power-of-two magnitude and sign, where the magnitude is - * the same for both operands. - */ - for (bt = 0; bt < max(dbits, sbits) - 1; bt++) /* Magnitude */ - for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ - for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) - for (sb = -(block2 / 2); - sb < (block2 + 1) / 2; sb++) { - s64 dst, src; - - dst = value(bt % dbits, db, sgn[k][0]); - src = value(bt % sbits, sb, sgn[k][1]); - i += (*emit)(self, arg, &insns[i], - dst, src); - } - - /* Append tail instructions */ - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insns[i++] = BPF_EXIT_INSN(); - BUG_ON(i > len); - - self->u.ptr.insns = insns; - self->u.ptr.len = i; - - return 0; -} - -/* - * Block size parameters used in pattern tests below. une as needed to - * increase/reduce the number combinations tested, see following examples. - * block values per operand MSB - * ---------------------------------------- - * 0 none - * 1 (1 << MSB) - * 2 (1 << MSB) + [-1, 0] - * 3 (1 << MSB) + [-1, 0, 1] - */ -#define PATTERN_BLOCK1 1 -#define PATTERN_BLOCK2 5 - -/* Number of test runs for a pattern test */ -#define NR_PATTERN_RUNS 1 - -/* - * Exhaustive tests of ALU operations for all combinations of power-of-two - * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the ALU and ALU64 operations for JITs that - * emit different code depending on the magnitude of the immediate value. - */ -static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int *a = arg; - int op = a[0]; - u32 flags = a[1]; - s16 off = (flags & F_SIGNED) ? 1 : 0; - int i = 0; - u64 res; - - if (!insns) - return 7; - - if (__bpf_alu_result(&res, dst, (s32)imm, op, flags)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_IMM_OFF(op, R1, imm, off); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int *a = arg; - int op = a[0]; - u32 flags = a[1]; - s16 off = (flags & F_SIGNED) ? 1 : 0; - int i = 0; - u64 res; - - if (!insns) - return 7; - - if (__bpf_alu_result(&res, dst, (s32)imm, op, flags | F_ALU32)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_IMM_OFF(op, R1, imm, off); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - int i = 0; - u64 res; - - if (!insns) - return 9; - - if (__bpf_alu_result(&res, dst, src, op, 0)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_REG(op, R1, R2); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - int i = 0; - u64 res; - - if (!insns) - return 9; - - if (__bpf_alu_result(&res, (u32)dst, (u32)src, op, 0)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_REG(op, R1, R2); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_fill_alu64_imm(struct bpf_test *self, int op, u32 flags) -{ - int arg[2] = {op, flags}; - - return __bpf_fill_pattern(self, &arg, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu64_imm); -} - -static int __bpf_fill_alu32_imm(struct bpf_test *self, int op, u32 flags) -{ - int arg[2] = {op, flags}; - - return __bpf_fill_pattern(self, &arg, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu32_imm); -} - -static int __bpf_fill_alu64_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu64_reg); -} - -static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu32_reg); -} - -/* ALU64 immediate operations */ -static int bpf_fill_alu64_mov_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOV, 0); -} - -static int bpf_fill_alu64_and_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_AND, 0); -} - -static int bpf_fill_alu64_or_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_OR, 0); -} - -static int bpf_fill_alu64_xor_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_XOR, 0); -} - -static int bpf_fill_alu64_add_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_ADD, 0); -} - -static int bpf_fill_alu64_sub_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_SUB, 0); -} - -static int bpf_fill_alu64_mul_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MUL, 0); -} - -static int bpf_fill_alu64_div_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_DIV, 0); -} - -static int bpf_fill_alu64_mod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOD, 0); -} - -/* Signed ALU64 immediate operations */ -static int bpf_fill_alu64_sdiv_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu64_smod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOD, F_SIGNED); -} - -/* Signed ALU32 immediate operations */ -static int bpf_fill_alu32_sdiv_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu32_smod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOD, F_SIGNED); -} - -/* ALU32 immediate operations */ -static int bpf_fill_alu32_mov_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOV, 0); -} - -static int bpf_fill_alu32_and_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_AND, 0); -} - -static int bpf_fill_alu32_or_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_OR, 0); -} - -static int bpf_fill_alu32_xor_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_XOR, 0); -} - -static int bpf_fill_alu32_add_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_ADD, 0); -} - -static int bpf_fill_alu32_sub_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_SUB, 0); -} - -static int bpf_fill_alu32_mul_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MUL, 0); -} - -static int bpf_fill_alu32_div_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_DIV, 0); -} - -static int bpf_fill_alu32_mod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOD, 0); -} - -/* ALU64 register operations */ -static int bpf_fill_alu64_mov_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MOV); -} - -static int bpf_fill_alu64_and_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_AND); -} - -static int bpf_fill_alu64_or_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_OR); -} - -static int bpf_fill_alu64_xor_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_XOR); -} - -static int bpf_fill_alu64_add_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_ADD); -} - -static int bpf_fill_alu64_sub_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_SUB); -} - -static int bpf_fill_alu64_mul_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MUL); -} - -static int bpf_fill_alu64_div_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_DIV); -} - -static int bpf_fill_alu64_mod_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MOD); -} - -/* ALU32 register operations */ -static int bpf_fill_alu32_mov_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MOV); -} - -static int bpf_fill_alu32_and_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_AND); -} - -static int bpf_fill_alu32_or_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_OR); -} - -static int bpf_fill_alu32_xor_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_XOR); -} - -static int bpf_fill_alu32_add_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_ADD); -} - -static int bpf_fill_alu32_sub_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_SUB); -} - -static int bpf_fill_alu32_mul_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MUL); -} - -static int bpf_fill_alu32_div_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_DIV); -} - -static int bpf_fill_alu32_mod_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MOD); -} - -/* - * Test JITs that implement complex ALU operations as function - * calls, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, - u32 flags) -{ - int len = 2 + 10 * 10; - struct bpf_insn *insns; - u64 dst, res; - int i = 0; - u32 imm; - int rd; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Operand and result values according to operation */ - if (flags & F_SIGNED) { - if (flags & F_ALU32) - dst = -76543210; - else - dst = -7654321076543210LL; - } else { - if (flags & F_ALU32) - dst = 0x76543210U; - else - dst = 0x7edcba9876543210ULL; - } - imm = 0x01234567U; - - if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - imm &= 31; - - __bpf_alu_result(&res, dst, imm, op, flags); - - if (flags & F_ALU32) - res = (u32)res; - - /* Check all operand registers */ - for (rd = R0; rd <= R9; rd++) { - i += __bpf_ld_imm64(&insns[i], rd, dst); - - s16 off = (flags & F_SIGNED) ? 1 : 0; - - if (flags & F_ALU32) - insns[i++] = BPF_ALU32_IMM_OFF(op, rd, imm, off); - else - insns[i++] = BPF_ALU64_IMM_OFF(op, rd, imm, off); - - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res >> 32, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - } - - insns[i++] = BPF_MOV64_IMM(R0, 1); - insns[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* ALU64 K registers */ -static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOV, 0); -} - -static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_AND, 0); -} - -static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_OR, 0); -} - -static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_XOR, 0); -} - -static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_LSH, 0); -} - -static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_RSH, 0); -} - -static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, 0); -} - -static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ADD, 0); -} - -static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_SUB, 0); -} - -static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MUL, 0); -} - -static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, 0); -} - -static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, 0); -} - -/* Signed ALU64 K registers */ -static int bpf_fill_alu64_sdiv_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu64_smod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_SIGNED); -} - -/* ALU32 K registers */ -static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOV, F_ALU32); -} - -static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_AND, F_ALU32); -} - -static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_OR, F_ALU32); -} - -static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_XOR, F_ALU32); -} - -static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_LSH, F_ALU32); -} - -static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_RSH, F_ALU32); -} - -static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, F_ALU32); -} - -static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ADD, F_ALU32); -} - -static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_SUB, F_ALU32); -} - -static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MUL, F_ALU32); -} - -static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32); -} - -static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32); -} - -/* Signed ALU32 K registers */ -static int bpf_fill_alu32_sdiv_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32 | F_SIGNED); -} - -static int bpf_fill_alu32_smod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32 | F_SIGNED); -} - -/* - * Test JITs that implement complex ALU operations as function - * calls, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) -{ - int len = 2 + 10 * 10 * 12; - u64 dst, src, res, same; - struct bpf_insn *insns; - int rd, rs; - int i = 0; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Operand and result values according to operation */ - if (alu32) { - dst = 0x76543210U; - src = 0x01234567U; - } else { - dst = 0x7edcba9876543210ULL; - src = 0x0123456789abcdefULL; - } - - if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - src &= 31; - - __bpf_alu_result(&res, dst, src, op, 0); - __bpf_alu_result(&same, src, src, op, 0); - - if (alu32) { - res = (u32)res; - same = (u32)same; - } - - /* Check all combinations of operand registers */ - for (rd = R0; rd <= R9; rd++) { - for (rs = R0; rs <= R9; rs++) { - u64 val = rd == rs ? same : res; - - i += __bpf_ld_imm64(&insns[i], rd, dst); - i += __bpf_ld_imm64(&insns[i], rs, src); - - if (alu32) - insns[i++] = BPF_ALU32_REG(op, rd, rs); - else - insns[i++] = BPF_ALU64_REG(op, rd, rs); - - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val >> 32, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - } - } - - insns[i++] = BPF_MOV64_IMM(R0, 1); - insns[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* ALU64 X register combinations */ -static int bpf_fill_alu64_mov_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOV, false); -} - -static int bpf_fill_alu64_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_AND, false); -} - -static int bpf_fill_alu64_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_OR, false); -} - -static int bpf_fill_alu64_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_XOR, false); -} - -static int bpf_fill_alu64_lsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_LSH, false); -} - -static int bpf_fill_alu64_rsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_RSH, false); -} - -static int bpf_fill_alu64_arsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, false); -} - -static int bpf_fill_alu64_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ADD, false); -} - -static int bpf_fill_alu64_sub_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_SUB, false); -} - -static int bpf_fill_alu64_mul_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MUL, false); -} - -static int bpf_fill_alu64_div_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_DIV, false); -} - -static int bpf_fill_alu64_mod_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOD, false); -} - -/* ALU32 X register combinations */ -static int bpf_fill_alu32_mov_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOV, true); -} - -static int bpf_fill_alu32_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_AND, true); -} - -static int bpf_fill_alu32_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_OR, true); -} - -static int bpf_fill_alu32_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_XOR, true); -} - -static int bpf_fill_alu32_lsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_LSH, true); -} - -static int bpf_fill_alu32_rsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_RSH, true); -} - -static int bpf_fill_alu32_arsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, true); -} - -static int bpf_fill_alu32_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ADD, true); -} - -static int bpf_fill_alu32_sub_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_SUB, true); -} - -static int bpf_fill_alu32_mul_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MUL, true); -} - -static int bpf_fill_alu32_div_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_DIV, true); -} - -static int bpf_fill_alu32_mod_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOD, true); -} - -/* - * Exhaustive tests of atomic operations for all power-of-two operand - * magnitudes, both for positive and negative values. - */ - -static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - u64 keep, fetch, res; - int i = 0; - - if (!insns) - return 21; - - switch (op) { - case BPF_XCHG: - res = src; - break; - default: - __bpf_alu_result(&res, dst, src, BPF_OP(op), 0); - } - - keep = 0x0123456789abcdefULL; - if (op & BPF_FETCH) - fetch = dst; - else - fetch = src; - - i += __bpf_ld_imm64(&insns[i], R0, keep); - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, res); - i += __bpf_ld_imm64(&insns[i], R4, fetch); - i += __bpf_ld_imm64(&insns[i], R5, keep); - - insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); - insns[i++] = BPF_ATOMIC_OP(BPF_DW, op, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R1, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - u64 keep, fetch, res; - int i = 0; - - if (!insns) - return 21; - - switch (op) { - case BPF_XCHG: - res = src; - break; - default: - __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op), 0); - } - - keep = 0x0123456789abcdefULL; - if (op & BPF_FETCH) - fetch = (u32)dst; - else - fetch = src; - - i += __bpf_ld_imm64(&insns[i], R0, keep); - i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - i += __bpf_ld_imm64(&insns[i], R4, fetch); - i += __bpf_ld_imm64(&insns[i], R5, keep); - - insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); - insns[i++] = BPF_ATOMIC_OP(BPF_W, op, R10, R2, -4); - insns[i++] = BPF_LDX_MEM(BPF_W, R1, R10, -4); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_cmpxchg64(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int i = 0; - - if (!insns) - return 23; - - i += __bpf_ld_imm64(&insns[i], R0, ~dst); - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - - /* Result unsuccessful */ - insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); - insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - /* Result successful */ - insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_cmpxchg32(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int i = 0; - - if (!insns) - return 27; - - i += __bpf_ld_imm64(&insns[i], R0, ~dst); - i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - - /* Result unsuccessful */ - insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); - insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); - insns[i++] = BPF_ZEXT_REG(R0); /* Zext always inserted by verifier */ - insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); - - insns[i++] = BPF_JMP32_REG(BPF_JEQ, R1, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - /* Result successful */ - i += __bpf_ld_imm64(&insns[i], R0, dst); - insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); - insns[i++] = BPF_ZEXT_REG(R0); /* Zext always inserted by verifier */ - insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); - - insns[i++] = BPF_JMP32_REG(BPF_JEQ, R2, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_fill_atomic64(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - 0, PATTERN_BLOCK2, - &__bpf_emit_atomic64); -} - -static int __bpf_fill_atomic32(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - 0, PATTERN_BLOCK2, - &__bpf_emit_atomic32); -} - -/* 64-bit atomic operations */ -static int bpf_fill_atomic64_add(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_ADD); -} - -static int bpf_fill_atomic64_and(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_AND); -} - -static int bpf_fill_atomic64_or(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_OR); -} - -static int bpf_fill_atomic64_xor(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XOR); -} - -static int bpf_fill_atomic64_add_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic64_and_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic64_or_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xor_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xchg(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XCHG); -} - -static int bpf_fill_cmpxchg64(struct bpf_test *self) -{ - return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, - &__bpf_emit_cmpxchg64); -} - -/* 32-bit atomic operations */ -static int bpf_fill_atomic32_add(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_ADD); -} - -static int bpf_fill_atomic32_and(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_AND); -} - -static int bpf_fill_atomic32_or(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_OR); -} - -static int bpf_fill_atomic32_xor(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XOR); -} - -static int bpf_fill_atomic32_add_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic32_and_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic32_or_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xor_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xchg(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XCHG); -} - -static int bpf_fill_cmpxchg32(struct bpf_test *self) -{ - return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, - &__bpf_emit_cmpxchg32); -} - -/* - * Test JITs that implement ATOMIC operations as function calls or - * other primitives, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) -{ - struct bpf_insn *insn; - int len = 2 + 34 * 10 * 10; - u64 mem, upd, res; - int rd, rs, i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - /* Operand and memory values */ - if (width == BPF_DW) { - mem = 0x0123456789abcdefULL; - upd = 0xfedcba9876543210ULL; - } else { /* BPF_W */ - mem = 0x01234567U; - upd = 0x76543210U; - } - - /* Memory updated according to operation */ - switch (op) { - case BPF_XCHG: - res = upd; - break; - case BPF_CMPXCHG: - res = mem; - break; - default: - __bpf_alu_result(&res, mem, upd, BPF_OP(op), 0); - } - - /* Test all operand registers */ - for (rd = R0; rd <= R9; rd++) { - for (rs = R0; rs <= R9; rs++) { - u64 cmp, src; - - /* Initialize value in memory */ - i += __bpf_ld_imm64(&insn[i], R0, mem); - insn[i++] = BPF_STX_MEM(width, R10, R0, -8); - - /* Initialize registers in order */ - i += __bpf_ld_imm64(&insn[i], R0, ~mem); - i += __bpf_ld_imm64(&insn[i], rs, upd); - insn[i++] = BPF_MOV64_REG(rd, R10); - - /* Perform atomic operation */ - insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8); - if (op == BPF_CMPXCHG && width == BPF_W) - insn[i++] = BPF_ZEXT_REG(R0); - - /* Check R0 register value */ - if (op == BPF_CMPXCHG) - cmp = mem; /* Expect value from memory */ - else if (R0 == rd || R0 == rs) - cmp = 0; /* Aliased, checked below */ - else - cmp = ~mem; /* Expect value to be preserved */ - if (cmp) { - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, - (u32)cmp, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32); - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, - cmp >> 32, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check source register value */ - if (rs == R0 && op == BPF_CMPXCHG) - src = 0; /* Aliased with R0, checked above */ - else if (rs == rd && (op == BPF_CMPXCHG || - !(op & BPF_FETCH))) - src = 0; /* Aliased with rd, checked below */ - else if (op == BPF_CMPXCHG) - src = upd; /* Expect value to be preserved */ - else if (op & BPF_FETCH) - src = mem; /* Expect fetched value from mem */ - else /* no fetch */ - src = upd; /* Expect value to be preserved */ - if (src) { - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, - (u32)src, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32); - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, - src >> 32, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check destination register value */ - if (!(rd == R0 && op == BPF_CMPXCHG) && - !(rd == rs && (op & BPF_FETCH))) { - insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check value in memory */ - if (rs != rd) { /* No aliasing */ - i += __bpf_ld_imm64(&insn[i], R1, res); - } else if (op == BPF_XCHG) { /* Aliased, XCHG */ - insn[i++] = BPF_MOV64_REG(R1, R10); - } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */ - i += __bpf_ld_imm64(&insn[i], R1, mem); - } else { /* Aliased, ALU oper */ - i += __bpf_ld_imm64(&insn[i], R1, mem); - insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10); - } - - insn[i++] = BPF_LDX_MEM(width, R0, R10, -8); - if (width == BPF_DW) - insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - else /* width == BPF_W */ - insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - } - - insn[i++] = BPF_MOV64_IMM(R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = i; - BUG_ON(i > len); - - return 0; -} - -/* 64-bit atomic register tests */ -static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD); -} - -static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND); -} - -static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR); -} - -static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR); -} - -static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG); -} - -static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG); -} - -/* 32-bit atomic register tests */ -static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD); -} - -static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND); -} - -static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR); -} - -static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR); -} - -static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG); -} - -static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG); -} - -/* - * Test the two-instruction 64-bit immediate load operation for all - * power-of-two magnitudes of the immediate operand. For each MSB, a block - * of immediate values centered around the power-of-two MSB are tested, - * both for positive and negative values. The test is designed to verify - * the operation for JITs that emit different code depending on the magnitude - * of the immediate value. This is often the case if the native instruction - * immediate field width is narrower than 32 bits. - */ -static int bpf_fill_ld_imm64_magn(struct bpf_test *self) -{ - int block = 64; /* Increase for more tests per MSB position */ - int len = 3 + 8 * 63 * block * 2; - struct bpf_insn *insn; - int bit, adj, sign; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (bit = 0; bit <= 62; bit++) { - for (adj = -block / 2; adj < block / 2; adj++) { - for (sign = -1; sign <= 1; sign += 2) { - s64 imm = sign * ((1LL << bit) + adj); - - /* Perform operation */ - i += __bpf_ld_imm64(&insn[i], R1, imm); - - /* Load reference */ - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, - (u32)(imm >> 32)); - insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); - insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); - - /* Check result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - } - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* - * Test the two-instruction 64-bit immediate load operation for different - * combinations of bytes. Each byte in the 64-bit word is constructed as - * (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG. - * All patterns (base1, mask1) and (base2, mask2) bytes are tested. - */ -static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self, - u8 base1, u8 mask1, - u8 base2, u8 mask2) -{ - struct bpf_insn *insn; - int len = 3 + 8 * BIT(8); - int pattern, index; - u32 rand = 1; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (pattern = 0; pattern < BIT(8); pattern++) { - u64 imm = 0; - - for (index = 0; index < 8; index++) { - int byte; - - if (pattern & BIT(index)) - byte = (base1 & mask1) | (rand & ~mask1); - else - byte = (base2 & mask2) | (rand & ~mask2); - imm = (imm << 8) | byte; - } - - /* Update our LCG */ - rand = rand * 1664525 + 1013904223; - - /* Perform operation */ - i += __bpf_ld_imm64(&insn[i], R1, imm); - - /* Load reference */ - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32)); - insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); - insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); - - /* Check result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_ld_imm64_checker(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff); -} - -static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80); -} - -static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff); -} - -static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff); -} - -/* - * Exhaustive tests of JMP operations for all combinations of power-of-two - * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the JMP and JMP32 operations for JITs that - * emit different code depending on the magnitude of the immediate value. - */ - -static bool __bpf_match_jmp_cond(s64 v1, s64 v2, u8 op) -{ - switch (op) { - case BPF_JSET: - return !!(v1 & v2); - case BPF_JEQ: - return v1 == v2; - case BPF_JNE: - return v1 != v2; - case BPF_JGT: - return (u64)v1 > (u64)v2; - case BPF_JGE: - return (u64)v1 >= (u64)v2; - case BPF_JLT: - return (u64)v1 < (u64)v2; - case BPF_JLE: - return (u64)v1 <= (u64)v2; - case BPF_JSGT: - return v1 > v2; - case BPF_JSGE: - return v1 >= v2; - case BPF_JSLT: - return v1 < v2; - case BPF_JSLE: - return v1 <= v2; - } - return false; -} - -static int __bpf_emit_jmp_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond(dst, (s32)imm, op); - int i = 0; - - insns[i++] = BPF_ALU32_IMM(BPF_MOV, R0, match); - - i += __bpf_ld_imm64(&insns[i], R1, dst); - insns[i++] = BPF_JMP_IMM(op, R1, imm, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 5 + 1; -} - -static int __bpf_emit_jmp32_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond((s32)dst, (s32)imm, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - insns[i++] = BPF_JMP32_IMM(op, R1, imm, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 5; -} - -static int __bpf_emit_jmp_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond(dst, src, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - insns[i++] = BPF_JMP_REG(op, R1, R2, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 7; -} - -static int __bpf_emit_jmp32_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond((s32)dst, (s32)src, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - insns[i++] = BPF_JMP32_REG(op, R1, R2, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 7; -} - -static int __bpf_fill_jmp_imm(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp_imm); -} - -static int __bpf_fill_jmp32_imm(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp32_imm); -} - -static int __bpf_fill_jmp_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp_reg); -} - -static int __bpf_fill_jmp32_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp32_reg); -} - -/* JMP immediate tests */ -static int bpf_fill_jmp_jset_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSET); -} - -static int bpf_fill_jmp_jeq_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JEQ); -} - -static int bpf_fill_jmp_jne_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JNE); -} - -static int bpf_fill_jmp_jgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JGT); -} - -static int bpf_fill_jmp_jge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JGE); -} - -static int bpf_fill_jmp_jlt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JLT); -} - -static int bpf_fill_jmp_jle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JLE); -} - -static int bpf_fill_jmp_jsgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSGT); -} - -static int bpf_fill_jmp_jsge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSGE); -} - -static int bpf_fill_jmp_jslt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSLT); -} - -static int bpf_fill_jmp_jsle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSLE); -} - -/* JMP32 immediate tests */ -static int bpf_fill_jmp32_jset_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSET); -} - -static int bpf_fill_jmp32_jeq_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JEQ); -} - -static int bpf_fill_jmp32_jne_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JNE); -} - -static int bpf_fill_jmp32_jgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JGT); -} - -static int bpf_fill_jmp32_jge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JGE); -} - -static int bpf_fill_jmp32_jlt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JLT); -} - -static int bpf_fill_jmp32_jle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JLE); -} - -static int bpf_fill_jmp32_jsgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSGT); -} - -static int bpf_fill_jmp32_jsge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSGE); -} - -static int bpf_fill_jmp32_jslt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSLT); -} - -static int bpf_fill_jmp32_jsle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSLE); -} - -/* JMP register tests */ -static int bpf_fill_jmp_jset_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSET); -} - -static int bpf_fill_jmp_jeq_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JEQ); -} - -static int bpf_fill_jmp_jne_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JNE); -} - -static int bpf_fill_jmp_jgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JGT); -} - -static int bpf_fill_jmp_jge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JGE); -} - -static int bpf_fill_jmp_jlt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JLT); -} - -static int bpf_fill_jmp_jle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JLE); -} - -static int bpf_fill_jmp_jsgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSGT); -} - -static int bpf_fill_jmp_jsge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSGE); -} - -static int bpf_fill_jmp_jslt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSLT); -} - -static int bpf_fill_jmp_jsle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSLE); -} - -/* JMP32 register tests */ -static int bpf_fill_jmp32_jset_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSET); -} - -static int bpf_fill_jmp32_jeq_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JEQ); -} - -static int bpf_fill_jmp32_jne_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JNE); -} - -static int bpf_fill_jmp32_jgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JGT); -} - -static int bpf_fill_jmp32_jge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JGE); -} - -static int bpf_fill_jmp32_jlt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JLT); -} - -static int bpf_fill_jmp32_jle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JLE); -} - -static int bpf_fill_jmp32_jsgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSGT); -} - -static int bpf_fill_jmp32_jsge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSGE); -} - -static int bpf_fill_jmp32_jslt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSLT); -} - -static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSLE); -} - -/* - * Set up a sequence of staggered jumps, forwards and backwards with - * increasing offset. This tests the conversion of relative jumps to - * JITed native jumps. On some architectures, for example MIPS, a large - * PC-relative jump offset may overflow the immediate field of the native - * conditional branch instruction, triggering a conversion to use an - * absolute jump instead. Since this changes the jump offsets, another - * offset computation pass is necessary, and that may in turn trigger - * another branch conversion. This jump sequence is particularly nasty - * in that regard. - * - * The sequence generation is parameterized by size and jump type. - * The size must be even, and the expected result is always size + 1. - * Below is an example with size=8 and result=9. - * - * ________________________Start - * R0 = 0 - * R1 = r1 - * R2 = r2 - * ,------- JMP +4 * 3______________Preamble: 4 insns - * ,----------|-ind 0- if R0 != 7 JMP 8 * 3 + 1 <--------------------. - * | | R0 = 8 | - * | | JMP +7 * 3 ------------------------. - * | ,--------|-----1- if R0 != 5 JMP 7 * 3 + 1 <--------------. | | - * | | | R0 = 6 | | | - * | | | JMP +5 * 3 ------------------. | | - * | | ,------|-----2- if R0 != 3 JMP 6 * 3 + 1 <--------. | | | | - * | | | | R0 = 4 | | | | | - * | | | | JMP +3 * 3 ------------. | | | | - * | | | ,----|-----3- if R0 != 1 JMP 5 * 3 + 1 <--. | | | | | | - * | | | | | R0 = 2 | | | | | | | - * | | | | | JMP +1 * 3 ------. | | | | | | - * | | | | ,--t=====4> if R0 != 0 JMP 4 * 3 + 1 1 2 3 4 5 6 7 8 loc - * | | | | | R0 = 1 -1 +2 -3 +4 -5 +6 -7 +8 off - * | | | | | JMP -2 * 3 ---' | | | | | | | - * | | | | | ,------5- if R0 != 2 JMP 3 * 3 + 1 <-----' | | | | | | - * | | | | | | R0 = 3 | | | | | | - * | | | | | | JMP -4 * 3 ---------' | | | | | - * | | | | | | ,----6- if R0 != 4 JMP 2 * 3 + 1 <-----------' | | | | - * | | | | | | | R0 = 5 | | | | - * | | | | | | | JMP -6 * 3 ---------------' | | | - * | | | | | | | ,--7- if R0 != 6 JMP 1 * 3 + 1 <-----------------' | | - * | | | | | | | | R0 = 7 | | - * | | Error | | | JMP -8 * 3 ---------------------' | - * | | paths | | | ,8- if R0 != 8 JMP 0 * 3 + 1 <-----------------------' - * | | | | | | | | | R0 = 9__________________Sequence: 3 * size - 1 insns - * `-+-+-+-+-+-+-+-+-> EXIT____________________Return: 1 insn - * - */ - -/* The maximum size parameter */ -#define MAX_STAGGERED_JMP_SIZE ((0x7fff / 3) & ~1) - -/* We use a reduced number of iterations to get a reasonable execution time */ -#define NR_STAGGERED_JMP_RUNS 10 - -static int __bpf_fill_staggered_jumps(struct bpf_test *self, - const struct bpf_insn *jmp, - u64 r1, u64 r2) -{ - int size = self->test[0].result - 1; - int len = 4 + 3 * (size + 1); - struct bpf_insn *insns; - int off, ind; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Preamble */ - insns[0] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - insns[1] = BPF_ALU64_IMM(BPF_MOV, R1, r1); - insns[2] = BPF_ALU64_IMM(BPF_MOV, R2, r2); - insns[3] = BPF_JMP_IMM(BPF_JA, 0, 0, 3 * size / 2); - - /* Sequence */ - for (ind = 0, off = size; ind <= size; ind++, off -= 2) { - struct bpf_insn *ins = &insns[4 + 3 * ind]; - int loc; - - if (off == 0) - off--; - - loc = abs(off); - ins[0] = BPF_JMP_IMM(BPF_JNE, R0, loc - 1, - 3 * (size - ind) + 1); - ins[1] = BPF_ALU64_IMM(BPF_MOV, R0, loc); - ins[2] = *jmp; - ins[2].off = 3 * (off - 1); - } - - /* Return */ - insns[len - 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - - return 0; -} - -/* 64-bit unconditional jump */ -static int bpf_fill_staggered_ja(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JA, 0, 0, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0, 0); -} - -/* 64-bit immediate jumps */ -static int bpf_fill_staggered_jeq_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JEQ, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jne_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JNE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); -} - -static int bpf_fill_staggered_jset_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSET, R1, 0x82, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); -} - -static int bpf_fill_staggered_jgt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGT, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); -} - -static int bpf_fill_staggered_jge_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jlt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLT, R1, 0x80000000, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jle_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jsgt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGT, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -static int bpf_fill_staggered_jsge_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGE, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jslt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLT, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jsle_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLE, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -/* 64-bit register jumps */ -static int bpf_fill_staggered_jeq_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JEQ, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jne_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JNE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); -} - -static int bpf_fill_staggered_jset_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSET, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); -} - -static int bpf_fill_staggered_jgt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); -} - -static int bpf_fill_staggered_jge_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jlt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); -} - -static int bpf_fill_staggered_jle_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jsgt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); -} - -static int bpf_fill_staggered_jsge_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); -} - -static int bpf_fill_staggered_jslt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); -} - -static int bpf_fill_staggered_jsle_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); -} - -/* 32-bit immediate jumps */ -static int bpf_fill_staggered_jeq32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JEQ, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jne32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JNE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); -} - -static int bpf_fill_staggered_jset32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSET, R1, 0x82, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); -} - -static int bpf_fill_staggered_jgt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGT, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); -} - -static int bpf_fill_staggered_jge32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jlt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLT, R1, 0x80000000, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jle32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jsgt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGT, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -static int bpf_fill_staggered_jsge32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGE, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jslt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLT, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jsle32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLE, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -/* 32-bit register jumps */ -static int bpf_fill_staggered_jeq32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JEQ, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jne32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JNE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); -} - -static int bpf_fill_staggered_jset32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSET, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); -} - -static int bpf_fill_staggered_jgt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); -} - -static int bpf_fill_staggered_jge32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jlt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); -} - -static int bpf_fill_staggered_jle32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jsgt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); -} - -static int bpf_fill_staggered_jsge32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); -} - -static int bpf_fill_staggered_jslt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); -} - -static int bpf_fill_staggered_jsle32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); -} - - -static struct bpf_test tests[] = { - { - "TAX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_ALU | BPF_NEG, 0), /* A == -3 */ - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), /* X == len - 3 */ - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 1), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 10, 20, 30, 40, 50 }, - { { 2, 10 }, { 3, 20 }, { 4, 30 } }, - }, - { - "TXA", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) /* A == len * 2 */ - }, - CLASSIC, - { 10, 20, 30, 40, 50 }, - { { 1, 2 }, { 3, 6 }, { 4, 8 } }, - }, - { - "ADD_SUB_MUL_K", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MUL | BPF_K, 3), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xfffffffd } } - }, - { - "DIV_MOD_KX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 8), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0x70000000), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_K, 0x70000000), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x20000000 } } - }, - { - "AND_OR_LSH_K", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xff), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_LSH | BPF_K, 27), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xf), - BPF_STMT(BPF_ALU | BPF_OR | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x800000ff }, { 1, 0x800000ff } }, - }, - { - "LD_IMM_0", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0), /* ld #0 */ - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - }, - CLASSIC, - { }, - { { 1, 1 } }, - }, - { - "LD_IND", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, MAX_K), - BPF_STMT(BPF_RET | BPF_K, 1) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 }, { 60, 0 } }, - }, - { - "LD_ABS", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, 1000), - BPF_STMT(BPF_RET | BPF_K, 1) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 }, { 60, 0 } }, - }, - { - "LD_ABS_LL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_LL_OFF), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_LL_OFF + 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 1, 2, 3 }, - { { 1, 0 }, { 2, 3 } }, - }, - { - "LD_IND_LL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, SKF_LL_OFF - 1), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 1, 2, 3, 0xff }, - { { 1, 1 }, { 3, 3 }, { 4, 0xff } }, - }, - { - "LD_ABS_NET", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_NET_OFF), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_NET_OFF + 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, - { { 15, 0 }, { 16, 3 } }, - }, - { - "LD_IND_NET", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, SKF_NET_OFF - 15), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, - { { 14, 0 }, { 15, 1 }, { 17, 3 } }, - }, - { - "LD_PKTTYPE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 3 }, { 10, 3 } }, - }, - { - "LD_MARK", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_MARK), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_MARK}, { 10, SKB_MARK} }, - }, - { - "LD_RXHASH", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_RXHASH), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_HASH}, { 10, SKB_HASH} }, - }, - { - "LD_QUEUE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_QUEUE), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_QUEUE_MAP }, { 10, SKB_QUEUE_MAP } }, - }, - { - "LD_PROTOCOL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 1), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 20, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PROTOCOL), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 30, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 10, 20, 30 }, - { { 10, ETH_P_IP }, { 100, ETH_P_IP } }, - }, - { - "LD_VLAN_TAG", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_VLAN_TAG), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_TCI }, - { 10, SKB_VLAN_TCI } - }, - }, - { - "LD_VLAN_TAG_PRESENT", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_PRESENT }, - { 10, SKB_VLAN_PRESENT } - }, - }, - { - "LD_IFINDEX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_IFINDEX), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_DEV_IFINDEX }, { 10, SKB_DEV_IFINDEX } }, - }, - { - "LD_HATYPE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_HATYPE), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_DEV_TYPE }, { 10, SKB_DEV_TYPE } }, - }, - { - "LD_CPU", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 } }, - }, - { - "LD_NLATTR", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 2), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, -#ifdef __BIG_ENDIAN - { 0xff, 0xff, 0, 4, 0, 2, 0, 4, 0, 3 }, -#else - { 0xff, 0xff, 4, 0, 2, 0, 4, 0, 3, 0 }, -#endif - { { 4, 0 }, { 20, 6 } }, - }, - { - "LD_NLATTR_NEST", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, -#ifdef __BIG_ENDIAN - { 0xff, 0xff, 0, 12, 0, 1, 0, 4, 0, 2, 0, 4, 0, 3 }, -#else - { 0xff, 0xff, 12, 0, 1, 0, 4, 0, 2, 0, 4, 0, 3, 0 }, -#endif - { { 4, 0 }, { 20, 10 } }, - }, - { - "LD_PAYLOAD_OFF", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - /* 00:00:00:00:00:00 > 00:00:00:00:00:00, ethtype IPv4 (0x0800), - * length 98: 127.0.0.1 > 127.0.0.1: ICMP echo request, - * id 9737, seq 1, length 64 - */ - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, - 0x45, 0x00, 0x00, 0x54, 0xac, 0x8b, 0x40, 0x00, 0x40, - 0x01, 0x90, 0x1b, 0x7f, 0x00, 0x00, 0x01 }, - { { 30, 0 }, { 100, 42 } }, - }, - { - "LD_ANC_XOR", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 10), - BPF_STMT(BPF_LDX | BPF_IMM, 300), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_ALU_XOR_X), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 4, 0xA ^ 300 }, { 20, 0xA ^ 300 } }, - }, - { - "SPILL_FILL", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_ALU | BPF_RSH, 1), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_ST, 1), /* M1 = 1 ^ len */ - BPF_STMT(BPF_ALU | BPF_XOR | BPF_K, 0x80000000), - BPF_STMT(BPF_ST, 2), /* M2 = 1 ^ len ^ 0x80000000 */ - BPF_STMT(BPF_STX, 15), /* M3 = len */ - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_LD | BPF_MEM, 2), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 0x80000001 }, { 2, 0x80000002 }, { 60, 0x80000000 ^ 60 } } - }, - { - "JEQ", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 3, 3, 3, 3, 3 }, - { { 1, 0 }, { 3, 1 }, { 4, MAX_K } }, - }, - { - "JGT", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 4, 3, 3 }, - { { 2, 0 }, { 3, 1 }, { 4, MAX_K } }, - }, - { - "JGE (jt 0), test 1", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 4, 3, 3 }, - { { 2, 0 }, { 3, 1 }, { 4, 1 } }, - }, - { - "JGE (jt 0), test 2", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 5, 3, 3 }, - { { 4, 1 }, { 5, 1 }, { 6, MAX_K } }, - }, - { - "JGE", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, MAX_K), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 1, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 10), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 2, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 20), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 3, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 4, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 40), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 1, 2, 3, 4, 5 }, - { { 1, 20 }, { 3, 40 }, { 5, MAX_K } }, - }, - { - "JSET", - .u.insns = { - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_JUMP(BPF_JMP | BPF_JA, 1, 1, 1), - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_K, 4), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_IND, 0), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 1, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 10), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x80000000, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 0, 0xAA, 0x55, 1 }, - { { 4, 10 }, { 5, 20 }, { 6, MAX_K } }, - }, - { - "tcpdump port 22", - .u.insns = { - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 12), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x86dd, 0, 8), /* IPv6 */ - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x84, 2, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x11, 0, 17), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 54), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 14, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 56), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 12, 13), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x0800, 0, 12), /* IPv4 */ - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 23), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x84, 2, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x11, 0, 8), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x1fff, 6, 0), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 14), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 2, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 16), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0xffff), - BPF_STMT(BPF_RET | BPF_K, 0), - }, - CLASSIC, - /* 3c:07:54:43:e5:76 > 10:bf:48:d6:43:d6, ethertype IPv4(0x0800) - * length 114: 10.1.1.149.49700 > 10.1.2.10.22: Flags [P.], - * seq 1305692979:1305693027, ack 3650467037, win 65535, - * options [nop,nop,TS val 2502645400 ecr 3971138], length 48 - */ - { 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6, - 0x3c, 0x07, 0x54, 0x43, 0xe5, 0x76, - 0x08, 0x00, - 0x45, 0x10, 0x00, 0x64, 0x75, 0xb5, - 0x40, 0x00, 0x40, 0x06, 0xad, 0x2e, /* IP header */ - 0x0a, 0x01, 0x01, 0x95, /* ip src */ - 0x0a, 0x01, 0x02, 0x0a, /* ip dst */ - 0xc2, 0x24, - 0x00, 0x16 /* dst port */ }, - { { 10, 0 }, { 30, 0 }, { 100, 65535 } }, - }, - { - "tcpdump complex", - .u.insns = { - /* tcpdump -nei eth0 'tcp port 22 and (((ip[2:2] - - * ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0) and - * (len > 115 or len < 30000000000)' -d - */ - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 12), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x86dd, 30, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x800, 0, 29), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 23), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 0, 27), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x1fff, 25, 0), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 14), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 2, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 16), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 0, 20), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 16), - BPF_STMT(BPF_ST, 1), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 14), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf), - BPF_STMT(BPF_ALU | BPF_LSH | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0x5), /* libpcap emits K on TAX */ - BPF_STMT(BPF_LD | BPF_MEM, 1), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_ST, 5), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 26), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_RSH | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0x9), /* libpcap emits K on TAX */ - BPF_STMT(BPF_LD | BPF_MEM, 5), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0, 4, 0), - BPF_STMT(BPF_LD | BPF_LEN, 0), - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, 0x73, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0xfc23ac00, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0xffff), - BPF_STMT(BPF_RET | BPF_K, 0), - }, - CLASSIC, - { 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6, - 0x3c, 0x07, 0x54, 0x43, 0xe5, 0x76, - 0x08, 0x00, - 0x45, 0x10, 0x00, 0x64, 0x75, 0xb5, - 0x40, 0x00, 0x40, 0x06, 0xad, 0x2e, /* IP header */ - 0x0a, 0x01, 0x01, 0x95, /* ip src */ - 0x0a, 0x01, 0x02, 0x0a, /* ip dst */ - 0xc2, 0x24, - 0x00, 0x16 /* dst port */ }, - { { 10, 0 }, { 30, 0 }, { 100, 65535 } }, - }, - { - "RET_A", - .u.insns = { - /* check that uninitialized X and A contain zeros */ - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { {1, 0}, {2, 0} }, - }, - { - "INT: ADD trivial", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_ALU64_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_IMM(BPF_ADD, R1, -1), - BPF_ALU64_IMM(BPF_MUL, R1, 3), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffd } } - }, - { - "INT: MUL_X", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU64_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_JMP_IMM(BPF_JEQ, R1, 0xfffffffd, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: MUL_X2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1), - BPF_ALU32_IMM(BPF_MOV, R1, -1), - BPF_ALU32_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_ALU64_IMM(BPF_RSH, R1, 8), - BPF_JMP_IMM(BPF_JEQ, R1, 0x2ffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: MUL32_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU32_IMM(BPF_MOV, R2, 3), - BPF_ALU32_REG(BPF_MUL, R1, R2), - BPF_ALU64_IMM(BPF_RSH, R1, 8), - BPF_JMP_IMM(BPF_JEQ, R1, 0xffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - /* Have to test all register combinations, since - * JITing of different registers will produce - * different asm code. - */ - "INT: ADD 64-bit", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_ADD, R0, 20), - BPF_ALU64_IMM(BPF_ADD, R1, 20), - BPF_ALU64_IMM(BPF_ADD, R2, 20), - BPF_ALU64_IMM(BPF_ADD, R3, 20), - BPF_ALU64_IMM(BPF_ADD, R4, 20), - BPF_ALU64_IMM(BPF_ADD, R5, 20), - BPF_ALU64_IMM(BPF_ADD, R6, 20), - BPF_ALU64_IMM(BPF_ADD, R7, 20), - BPF_ALU64_IMM(BPF_ADD, R8, 20), - BPF_ALU64_IMM(BPF_ADD, R9, 20), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_ALU64_IMM(BPF_SUB, R1, 10), - BPF_ALU64_IMM(BPF_SUB, R2, 10), - BPF_ALU64_IMM(BPF_SUB, R3, 10), - BPF_ALU64_IMM(BPF_SUB, R4, 10), - BPF_ALU64_IMM(BPF_SUB, R5, 10), - BPF_ALU64_IMM(BPF_SUB, R6, 10), - BPF_ALU64_IMM(BPF_SUB, R7, 10), - BPF_ALU64_IMM(BPF_SUB, R8, 10), - BPF_ALU64_IMM(BPF_SUB, R9, 10), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), /* R0 == 155 */ - BPF_JMP_IMM(BPF_JEQ, R0, 155, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R1, R0), - BPF_ALU64_REG(BPF_ADD, R1, R1), - BPF_ALU64_REG(BPF_ADD, R1, R2), - BPF_ALU64_REG(BPF_ADD, R1, R3), - BPF_ALU64_REG(BPF_ADD, R1, R4), - BPF_ALU64_REG(BPF_ADD, R1, R5), - BPF_ALU64_REG(BPF_ADD, R1, R6), - BPF_ALU64_REG(BPF_ADD, R1, R7), - BPF_ALU64_REG(BPF_ADD, R1, R8), - BPF_ALU64_REG(BPF_ADD, R1, R9), /* R1 == 456 */ - BPF_JMP_IMM(BPF_JEQ, R1, 456, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R2, R0), - BPF_ALU64_REG(BPF_ADD, R2, R1), - BPF_ALU64_REG(BPF_ADD, R2, R2), - BPF_ALU64_REG(BPF_ADD, R2, R3), - BPF_ALU64_REG(BPF_ADD, R2, R4), - BPF_ALU64_REG(BPF_ADD, R2, R5), - BPF_ALU64_REG(BPF_ADD, R2, R6), - BPF_ALU64_REG(BPF_ADD, R2, R7), - BPF_ALU64_REG(BPF_ADD, R2, R8), - BPF_ALU64_REG(BPF_ADD, R2, R9), /* R2 == 1358 */ - BPF_JMP_IMM(BPF_JEQ, R2, 1358, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R3, R0), - BPF_ALU64_REG(BPF_ADD, R3, R1), - BPF_ALU64_REG(BPF_ADD, R3, R2), - BPF_ALU64_REG(BPF_ADD, R3, R3), - BPF_ALU64_REG(BPF_ADD, R3, R4), - BPF_ALU64_REG(BPF_ADD, R3, R5), - BPF_ALU64_REG(BPF_ADD, R3, R6), - BPF_ALU64_REG(BPF_ADD, R3, R7), - BPF_ALU64_REG(BPF_ADD, R3, R8), - BPF_ALU64_REG(BPF_ADD, R3, R9), /* R3 == 4063 */ - BPF_JMP_IMM(BPF_JEQ, R3, 4063, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R4, R0), - BPF_ALU64_REG(BPF_ADD, R4, R1), - BPF_ALU64_REG(BPF_ADD, R4, R2), - BPF_ALU64_REG(BPF_ADD, R4, R3), - BPF_ALU64_REG(BPF_ADD, R4, R4), - BPF_ALU64_REG(BPF_ADD, R4, R5), - BPF_ALU64_REG(BPF_ADD, R4, R6), - BPF_ALU64_REG(BPF_ADD, R4, R7), - BPF_ALU64_REG(BPF_ADD, R4, R8), - BPF_ALU64_REG(BPF_ADD, R4, R9), /* R4 == 12177 */ - BPF_JMP_IMM(BPF_JEQ, R4, 12177, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R5, R0), - BPF_ALU64_REG(BPF_ADD, R5, R1), - BPF_ALU64_REG(BPF_ADD, R5, R2), - BPF_ALU64_REG(BPF_ADD, R5, R3), - BPF_ALU64_REG(BPF_ADD, R5, R4), - BPF_ALU64_REG(BPF_ADD, R5, R5), - BPF_ALU64_REG(BPF_ADD, R5, R6), - BPF_ALU64_REG(BPF_ADD, R5, R7), - BPF_ALU64_REG(BPF_ADD, R5, R8), - BPF_ALU64_REG(BPF_ADD, R5, R9), /* R5 == 36518 */ - BPF_JMP_IMM(BPF_JEQ, R5, 36518, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R6, R0), - BPF_ALU64_REG(BPF_ADD, R6, R1), - BPF_ALU64_REG(BPF_ADD, R6, R2), - BPF_ALU64_REG(BPF_ADD, R6, R3), - BPF_ALU64_REG(BPF_ADD, R6, R4), - BPF_ALU64_REG(BPF_ADD, R6, R5), - BPF_ALU64_REG(BPF_ADD, R6, R6), - BPF_ALU64_REG(BPF_ADD, R6, R7), - BPF_ALU64_REG(BPF_ADD, R6, R8), - BPF_ALU64_REG(BPF_ADD, R6, R9), /* R6 == 109540 */ - BPF_JMP_IMM(BPF_JEQ, R6, 109540, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R7, R0), - BPF_ALU64_REG(BPF_ADD, R7, R1), - BPF_ALU64_REG(BPF_ADD, R7, R2), - BPF_ALU64_REG(BPF_ADD, R7, R3), - BPF_ALU64_REG(BPF_ADD, R7, R4), - BPF_ALU64_REG(BPF_ADD, R7, R5), - BPF_ALU64_REG(BPF_ADD, R7, R6), - BPF_ALU64_REG(BPF_ADD, R7, R7), - BPF_ALU64_REG(BPF_ADD, R7, R8), - BPF_ALU64_REG(BPF_ADD, R7, R9), /* R7 == 328605 */ - BPF_JMP_IMM(BPF_JEQ, R7, 328605, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R8, R0), - BPF_ALU64_REG(BPF_ADD, R8, R1), - BPF_ALU64_REG(BPF_ADD, R8, R2), - BPF_ALU64_REG(BPF_ADD, R8, R3), - BPF_ALU64_REG(BPF_ADD, R8, R4), - BPF_ALU64_REG(BPF_ADD, R8, R5), - BPF_ALU64_REG(BPF_ADD, R8, R6), - BPF_ALU64_REG(BPF_ADD, R8, R7), - BPF_ALU64_REG(BPF_ADD, R8, R8), - BPF_ALU64_REG(BPF_ADD, R8, R9), /* R8 == 985799 */ - BPF_JMP_IMM(BPF_JEQ, R8, 985799, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R9, R0), - BPF_ALU64_REG(BPF_ADD, R9, R1), - BPF_ALU64_REG(BPF_ADD, R9, R2), - BPF_ALU64_REG(BPF_ADD, R9, R3), - BPF_ALU64_REG(BPF_ADD, R9, R4), - BPF_ALU64_REG(BPF_ADD, R9, R5), - BPF_ALU64_REG(BPF_ADD, R9, R6), - BPF_ALU64_REG(BPF_ADD, R9, R7), - BPF_ALU64_REG(BPF_ADD, R9, R8), - BPF_ALU64_REG(BPF_ADD, R9, R9), /* R9 == 2957380 */ - BPF_ALU64_REG(BPF_MOV, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2957380 } } - }, - { - "INT: ADD 32-bit", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 20), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R2, 2), - BPF_ALU32_IMM(BPF_MOV, R3, 3), - BPF_ALU32_IMM(BPF_MOV, R4, 4), - BPF_ALU32_IMM(BPF_MOV, R5, 5), - BPF_ALU32_IMM(BPF_MOV, R6, 6), - BPF_ALU32_IMM(BPF_MOV, R7, 7), - BPF_ALU32_IMM(BPF_MOV, R8, 8), - BPF_ALU32_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_ADD, R1, 10), - BPF_ALU64_IMM(BPF_ADD, R2, 10), - BPF_ALU64_IMM(BPF_ADD, R3, 10), - BPF_ALU64_IMM(BPF_ADD, R4, 10), - BPF_ALU64_IMM(BPF_ADD, R5, 10), - BPF_ALU64_IMM(BPF_ADD, R6, 10), - BPF_ALU64_IMM(BPF_ADD, R7, 10), - BPF_ALU64_IMM(BPF_ADD, R8, 10), - BPF_ALU64_IMM(BPF_ADD, R9, 10), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_ALU32_REG(BPF_ADD, R0, R2), - BPF_ALU32_REG(BPF_ADD, R0, R3), - BPF_ALU32_REG(BPF_ADD, R0, R4), - BPF_ALU32_REG(BPF_ADD, R0, R5), - BPF_ALU32_REG(BPF_ADD, R0, R6), - BPF_ALU32_REG(BPF_ADD, R0, R7), - BPF_ALU32_REG(BPF_ADD, R0, R8), - BPF_ALU32_REG(BPF_ADD, R0, R9), /* R0 == 155 */ - BPF_JMP_IMM(BPF_JEQ, R0, 155, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R1, R0), - BPF_ALU32_REG(BPF_ADD, R1, R1), - BPF_ALU32_REG(BPF_ADD, R1, R2), - BPF_ALU32_REG(BPF_ADD, R1, R3), - BPF_ALU32_REG(BPF_ADD, R1, R4), - BPF_ALU32_REG(BPF_ADD, R1, R5), - BPF_ALU32_REG(BPF_ADD, R1, R6), - BPF_ALU32_REG(BPF_ADD, R1, R7), - BPF_ALU32_REG(BPF_ADD, R1, R8), - BPF_ALU32_REG(BPF_ADD, R1, R9), /* R1 == 456 */ - BPF_JMP_IMM(BPF_JEQ, R1, 456, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R2, R0), - BPF_ALU32_REG(BPF_ADD, R2, R1), - BPF_ALU32_REG(BPF_ADD, R2, R2), - BPF_ALU32_REG(BPF_ADD, R2, R3), - BPF_ALU32_REG(BPF_ADD, R2, R4), - BPF_ALU32_REG(BPF_ADD, R2, R5), - BPF_ALU32_REG(BPF_ADD, R2, R6), - BPF_ALU32_REG(BPF_ADD, R2, R7), - BPF_ALU32_REG(BPF_ADD, R2, R8), - BPF_ALU32_REG(BPF_ADD, R2, R9), /* R2 == 1358 */ - BPF_JMP_IMM(BPF_JEQ, R2, 1358, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R3, R0), - BPF_ALU32_REG(BPF_ADD, R3, R1), - BPF_ALU32_REG(BPF_ADD, R3, R2), - BPF_ALU32_REG(BPF_ADD, R3, R3), - BPF_ALU32_REG(BPF_ADD, R3, R4), - BPF_ALU32_REG(BPF_ADD, R3, R5), - BPF_ALU32_REG(BPF_ADD, R3, R6), - BPF_ALU32_REG(BPF_ADD, R3, R7), - BPF_ALU32_REG(BPF_ADD, R3, R8), - BPF_ALU32_REG(BPF_ADD, R3, R9), /* R3 == 4063 */ - BPF_JMP_IMM(BPF_JEQ, R3, 4063, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R4, R0), - BPF_ALU32_REG(BPF_ADD, R4, R1), - BPF_ALU32_REG(BPF_ADD, R4, R2), - BPF_ALU32_REG(BPF_ADD, R4, R3), - BPF_ALU32_REG(BPF_ADD, R4, R4), - BPF_ALU32_REG(BPF_ADD, R4, R5), - BPF_ALU32_REG(BPF_ADD, R4, R6), - BPF_ALU32_REG(BPF_ADD, R4, R7), - BPF_ALU32_REG(BPF_ADD, R4, R8), - BPF_ALU32_REG(BPF_ADD, R4, R9), /* R4 == 12177 */ - BPF_JMP_IMM(BPF_JEQ, R4, 12177, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R5, R0), - BPF_ALU32_REG(BPF_ADD, R5, R1), - BPF_ALU32_REG(BPF_ADD, R5, R2), - BPF_ALU32_REG(BPF_ADD, R5, R3), - BPF_ALU32_REG(BPF_ADD, R5, R4), - BPF_ALU32_REG(BPF_ADD, R5, R5), - BPF_ALU32_REG(BPF_ADD, R5, R6), - BPF_ALU32_REG(BPF_ADD, R5, R7), - BPF_ALU32_REG(BPF_ADD, R5, R8), - BPF_ALU32_REG(BPF_ADD, R5, R9), /* R5 == 36518 */ - BPF_JMP_IMM(BPF_JEQ, R5, 36518, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R6, R0), - BPF_ALU32_REG(BPF_ADD, R6, R1), - BPF_ALU32_REG(BPF_ADD, R6, R2), - BPF_ALU32_REG(BPF_ADD, R6, R3), - BPF_ALU32_REG(BPF_ADD, R6, R4), - BPF_ALU32_REG(BPF_ADD, R6, R5), - BPF_ALU32_REG(BPF_ADD, R6, R6), - BPF_ALU32_REG(BPF_ADD, R6, R7), - BPF_ALU32_REG(BPF_ADD, R6, R8), - BPF_ALU32_REG(BPF_ADD, R6, R9), /* R6 == 109540 */ - BPF_JMP_IMM(BPF_JEQ, R6, 109540, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R7, R0), - BPF_ALU32_REG(BPF_ADD, R7, R1), - BPF_ALU32_REG(BPF_ADD, R7, R2), - BPF_ALU32_REG(BPF_ADD, R7, R3), - BPF_ALU32_REG(BPF_ADD, R7, R4), - BPF_ALU32_REG(BPF_ADD, R7, R5), - BPF_ALU32_REG(BPF_ADD, R7, R6), - BPF_ALU32_REG(BPF_ADD, R7, R7), - BPF_ALU32_REG(BPF_ADD, R7, R8), - BPF_ALU32_REG(BPF_ADD, R7, R9), /* R7 == 328605 */ - BPF_JMP_IMM(BPF_JEQ, R7, 328605, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R8, R0), - BPF_ALU32_REG(BPF_ADD, R8, R1), - BPF_ALU32_REG(BPF_ADD, R8, R2), - BPF_ALU32_REG(BPF_ADD, R8, R3), - BPF_ALU32_REG(BPF_ADD, R8, R4), - BPF_ALU32_REG(BPF_ADD, R8, R5), - BPF_ALU32_REG(BPF_ADD, R8, R6), - BPF_ALU32_REG(BPF_ADD, R8, R7), - BPF_ALU32_REG(BPF_ADD, R8, R8), - BPF_ALU32_REG(BPF_ADD, R8, R9), /* R8 == 985799 */ - BPF_JMP_IMM(BPF_JEQ, R8, 985799, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R9, R0), - BPF_ALU32_REG(BPF_ADD, R9, R1), - BPF_ALU32_REG(BPF_ADD, R9, R2), - BPF_ALU32_REG(BPF_ADD, R9, R3), - BPF_ALU32_REG(BPF_ADD, R9, R4), - BPF_ALU32_REG(BPF_ADD, R9, R5), - BPF_ALU32_REG(BPF_ADD, R9, R6), - BPF_ALU32_REG(BPF_ADD, R9, R7), - BPF_ALU32_REG(BPF_ADD, R9, R8), - BPF_ALU32_REG(BPF_ADD, R9, R9), /* R9 == 2957380 */ - BPF_ALU32_REG(BPF_MOV, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2957380 } } - }, - { /* Mainly checking JIT here. */ - "INT: SUB", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_REG(BPF_SUB, R0, R0), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_ALU64_REG(BPF_SUB, R0, R3), - BPF_ALU64_REG(BPF_SUB, R0, R4), - BPF_ALU64_REG(BPF_SUB, R0, R5), - BPF_ALU64_REG(BPF_SUB, R0, R6), - BPF_ALU64_REG(BPF_SUB, R0, R7), - BPF_ALU64_REG(BPF_SUB, R0, R8), - BPF_ALU64_REG(BPF_SUB, R0, R9), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_JMP_IMM(BPF_JEQ, R0, -55, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R1, R0), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_REG(BPF_SUB, R1, R3), - BPF_ALU64_REG(BPF_SUB, R1, R4), - BPF_ALU64_REG(BPF_SUB, R1, R5), - BPF_ALU64_REG(BPF_SUB, R1, R6), - BPF_ALU64_REG(BPF_SUB, R1, R7), - BPF_ALU64_REG(BPF_SUB, R1, R8), - BPF_ALU64_REG(BPF_SUB, R1, R9), - BPF_ALU64_IMM(BPF_SUB, R1, 10), - BPF_ALU64_REG(BPF_SUB, R2, R0), - BPF_ALU64_REG(BPF_SUB, R2, R1), - BPF_ALU64_REG(BPF_SUB, R2, R3), - BPF_ALU64_REG(BPF_SUB, R2, R4), - BPF_ALU64_REG(BPF_SUB, R2, R5), - BPF_ALU64_REG(BPF_SUB, R2, R6), - BPF_ALU64_REG(BPF_SUB, R2, R7), - BPF_ALU64_REG(BPF_SUB, R2, R8), - BPF_ALU64_REG(BPF_SUB, R2, R9), - BPF_ALU64_IMM(BPF_SUB, R2, 10), - BPF_ALU64_REG(BPF_SUB, R3, R0), - BPF_ALU64_REG(BPF_SUB, R3, R1), - BPF_ALU64_REG(BPF_SUB, R3, R2), - BPF_ALU64_REG(BPF_SUB, R3, R4), - BPF_ALU64_REG(BPF_SUB, R3, R5), - BPF_ALU64_REG(BPF_SUB, R3, R6), - BPF_ALU64_REG(BPF_SUB, R3, R7), - BPF_ALU64_REG(BPF_SUB, R3, R8), - BPF_ALU64_REG(BPF_SUB, R3, R9), - BPF_ALU64_IMM(BPF_SUB, R3, 10), - BPF_ALU64_REG(BPF_SUB, R4, R0), - BPF_ALU64_REG(BPF_SUB, R4, R1), - BPF_ALU64_REG(BPF_SUB, R4, R2), - BPF_ALU64_REG(BPF_SUB, R4, R3), - BPF_ALU64_REG(BPF_SUB, R4, R5), - BPF_ALU64_REG(BPF_SUB, R4, R6), - BPF_ALU64_REG(BPF_SUB, R4, R7), - BPF_ALU64_REG(BPF_SUB, R4, R8), - BPF_ALU64_REG(BPF_SUB, R4, R9), - BPF_ALU64_IMM(BPF_SUB, R4, 10), - BPF_ALU64_REG(BPF_SUB, R5, R0), - BPF_ALU64_REG(BPF_SUB, R5, R1), - BPF_ALU64_REG(BPF_SUB, R5, R2), - BPF_ALU64_REG(BPF_SUB, R5, R3), - BPF_ALU64_REG(BPF_SUB, R5, R4), - BPF_ALU64_REG(BPF_SUB, R5, R6), - BPF_ALU64_REG(BPF_SUB, R5, R7), - BPF_ALU64_REG(BPF_SUB, R5, R8), - BPF_ALU64_REG(BPF_SUB, R5, R9), - BPF_ALU64_IMM(BPF_SUB, R5, 10), - BPF_ALU64_REG(BPF_SUB, R6, R0), - BPF_ALU64_REG(BPF_SUB, R6, R1), - BPF_ALU64_REG(BPF_SUB, R6, R2), - BPF_ALU64_REG(BPF_SUB, R6, R3), - BPF_ALU64_REG(BPF_SUB, R6, R4), - BPF_ALU64_REG(BPF_SUB, R6, R5), - BPF_ALU64_REG(BPF_SUB, R6, R7), - BPF_ALU64_REG(BPF_SUB, R6, R8), - BPF_ALU64_REG(BPF_SUB, R6, R9), - BPF_ALU64_IMM(BPF_SUB, R6, 10), - BPF_ALU64_REG(BPF_SUB, R7, R0), - BPF_ALU64_REG(BPF_SUB, R7, R1), - BPF_ALU64_REG(BPF_SUB, R7, R2), - BPF_ALU64_REG(BPF_SUB, R7, R3), - BPF_ALU64_REG(BPF_SUB, R7, R4), - BPF_ALU64_REG(BPF_SUB, R7, R5), - BPF_ALU64_REG(BPF_SUB, R7, R6), - BPF_ALU64_REG(BPF_SUB, R7, R8), - BPF_ALU64_REG(BPF_SUB, R7, R9), - BPF_ALU64_IMM(BPF_SUB, R7, 10), - BPF_ALU64_REG(BPF_SUB, R8, R0), - BPF_ALU64_REG(BPF_SUB, R8, R1), - BPF_ALU64_REG(BPF_SUB, R8, R2), - BPF_ALU64_REG(BPF_SUB, R8, R3), - BPF_ALU64_REG(BPF_SUB, R8, R4), - BPF_ALU64_REG(BPF_SUB, R8, R5), - BPF_ALU64_REG(BPF_SUB, R8, R6), - BPF_ALU64_REG(BPF_SUB, R8, R7), - BPF_ALU64_REG(BPF_SUB, R8, R9), - BPF_ALU64_IMM(BPF_SUB, R8, 10), - BPF_ALU64_REG(BPF_SUB, R9, R0), - BPF_ALU64_REG(BPF_SUB, R9, R1), - BPF_ALU64_REG(BPF_SUB, R9, R2), - BPF_ALU64_REG(BPF_SUB, R9, R3), - BPF_ALU64_REG(BPF_SUB, R9, R4), - BPF_ALU64_REG(BPF_SUB, R9, R5), - BPF_ALU64_REG(BPF_SUB, R9, R6), - BPF_ALU64_REG(BPF_SUB, R9, R7), - BPF_ALU64_REG(BPF_SUB, R9, R8), - BPF_ALU64_IMM(BPF_SUB, R9, 10), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_ALU64_REG(BPF_SUB, R0, R3), - BPF_ALU64_REG(BPF_SUB, R0, R4), - BPF_ALU64_REG(BPF_SUB, R0, R5), - BPF_ALU64_REG(BPF_SUB, R0, R6), - BPF_ALU64_REG(BPF_SUB, R0, R7), - BPF_ALU64_REG(BPF_SUB, R0, R8), - BPF_ALU64_REG(BPF_SUB, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 11 } } - }, - { /* Mainly checking JIT here. */ - "INT: XOR", - .u.insns_int = { - BPF_ALU64_REG(BPF_SUB, R0, R0), - BPF_ALU64_REG(BPF_XOR, R1, R1), - BPF_JMP_REG(BPF_JEQ, R0, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 10), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU64_REG(BPF_SUB, R1, R1), - BPF_ALU64_REG(BPF_XOR, R2, R2), - BPF_JMP_REG(BPF_JEQ, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R2, R2), - BPF_ALU64_REG(BPF_XOR, R3, R3), - BPF_ALU64_IMM(BPF_MOV, R0, 10), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_JMP_REG(BPF_JEQ, R2, R3, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R3, R3), - BPF_ALU64_REG(BPF_XOR, R4, R4), - BPF_ALU64_IMM(BPF_MOV, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R5, -1), - BPF_JMP_REG(BPF_JEQ, R3, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R4, R4), - BPF_ALU64_REG(BPF_XOR, R5, R5), - BPF_ALU64_IMM(BPF_MOV, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R7, -1), - BPF_JMP_REG(BPF_JEQ, R5, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R5, 1), - BPF_ALU64_REG(BPF_SUB, R5, R5), - BPF_ALU64_REG(BPF_XOR, R6, R6), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R8, -1), - BPF_JMP_REG(BPF_JEQ, R5, R6, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R6, R6), - BPF_ALU64_REG(BPF_XOR, R7, R7), - BPF_JMP_REG(BPF_JEQ, R7, R6, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R7, R7), - BPF_ALU64_REG(BPF_XOR, R8, R8), - BPF_JMP_REG(BPF_JEQ, R7, R8, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R8, R8), - BPF_ALU64_REG(BPF_XOR, R9, R9), - BPF_JMP_REG(BPF_JEQ, R9, R8, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R9, R9), - BPF_ALU64_REG(BPF_XOR, R0, R0), - BPF_JMP_REG(BPF_JEQ, R9, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R1, R1), - BPF_ALU64_REG(BPF_XOR, R0, R0), - BPF_JMP_REG(BPF_JEQ, R9, R0, 2), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { /* Mainly checking JIT here. */ - "INT: MUL", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 11), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_REG(BPF_MUL, R0, R0), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_ALU64_REG(BPF_MUL, R0, R2), - BPF_ALU64_REG(BPF_MUL, R0, R3), - BPF_ALU64_REG(BPF_MUL, R0, R4), - BPF_ALU64_REG(BPF_MUL, R0, R5), - BPF_ALU64_REG(BPF_MUL, R0, R6), - BPF_ALU64_REG(BPF_MUL, R0, R7), - BPF_ALU64_REG(BPF_MUL, R0, R8), - BPF_ALU64_REG(BPF_MUL, R0, R9), - BPF_ALU64_IMM(BPF_MUL, R0, 10), - BPF_JMP_IMM(BPF_JEQ, R0, 439084800, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_MUL, R1, R0), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_ALU64_REG(BPF_MUL, R1, R3), - BPF_ALU64_REG(BPF_MUL, R1, R4), - BPF_ALU64_REG(BPF_MUL, R1, R5), - BPF_ALU64_REG(BPF_MUL, R1, R6), - BPF_ALU64_REG(BPF_MUL, R1, R7), - BPF_ALU64_REG(BPF_MUL, R1, R8), - BPF_ALU64_REG(BPF_MUL, R1, R9), - BPF_ALU64_IMM(BPF_MUL, R1, 10), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_JMP_IMM(BPF_JEQ, R2, 0x5a924, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_LSH, R1, 32), - BPF_ALU64_IMM(BPF_ARSH, R1, 32), - BPF_JMP_IMM(BPF_JEQ, R1, 0xebb90000, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_MUL, R2, R0), - BPF_ALU64_REG(BPF_MUL, R2, R1), - BPF_ALU64_REG(BPF_MUL, R2, R3), - BPF_ALU64_REG(BPF_MUL, R2, R4), - BPF_ALU64_REG(BPF_MUL, R2, R5), - BPF_ALU64_REG(BPF_MUL, R2, R6), - BPF_ALU64_REG(BPF_MUL, R2, R7), - BPF_ALU64_REG(BPF_MUL, R2, R8), - BPF_ALU64_REG(BPF_MUL, R2, R9), - BPF_ALU64_IMM(BPF_MUL, R2, 10), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_REG(BPF_MOV, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x35d97ef2 } } - }, - { /* Mainly checking JIT here. */ - "MOV REG64", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 0), - BPF_ALU64_IMM(BPF_MOV, R2, 0), - BPF_ALU64_IMM(BPF_MOV, R3, 0), - BPF_ALU64_IMM(BPF_MOV, R4, 0), - BPF_ALU64_IMM(BPF_MOV, R5, 0), - BPF_ALU64_IMM(BPF_MOV, R6, 0), - BPF_ALU64_IMM(BPF_MOV, R7, 0), - BPF_ALU64_IMM(BPF_MOV, R8, 0), - BPF_ALU64_IMM(BPF_MOV, R9, 0), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { /* Mainly checking JIT here. */ - "MOV REG32", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU32_IMM(BPF_MOV, R2, 0), - BPF_ALU32_IMM(BPF_MOV, R3, 0), - BPF_ALU32_IMM(BPF_MOV, R4, 0), - BPF_ALU32_IMM(BPF_MOV, R5, 0), - BPF_ALU32_IMM(BPF_MOV, R6, 0), - BPF_ALU32_IMM(BPF_MOV, R7, 0), - BPF_ALU32_IMM(BPF_MOV, R8, 0), - BPF_ALU32_IMM(BPF_MOV, R9, 0), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { /* Mainly checking JIT here. */ - "LD IMM64", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_LD_IMM64(R0, 0x0LL), - BPF_LD_IMM64(R1, 0x0LL), - BPF_LD_IMM64(R2, 0x0LL), - BPF_LD_IMM64(R3, 0x0LL), - BPF_LD_IMM64(R4, 0x0LL), - BPF_LD_IMM64(R5, 0x0LL), - BPF_LD_IMM64(R6, 0x0LL), - BPF_LD_IMM64(R7, 0x0LL), - BPF_LD_IMM64(R8, 0x0LL), - BPF_LD_IMM64(R9, 0x0LL), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { - "INT: ALU MIX", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 11), - BPF_ALU64_IMM(BPF_ADD, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_XOR, R2, 3), - BPF_ALU64_REG(BPF_DIV, R0, R2), - BPF_JMP_IMM(BPF_JEQ, R0, 10, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOD, R0, 3), - BPF_JMP_IMM(BPF_JEQ, R0, 1, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "INT: shifts by register", - .u.insns_int = { - BPF_MOV64_IMM(R0, -1234), - BPF_MOV64_IMM(R1, 1), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, 0x7ffffd97, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R2, 1), - BPF_ALU64_REG(BPF_LSH, R0, R2), - BPF_MOV32_IMM(R4, -1234), - BPF_JMP_REG(BPF_JEQ, R0, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_AND, R4, 63), - BPF_ALU64_REG(BPF_LSH, R0, R4), /* R0 <= 46 */ - BPF_MOV64_IMM(R3, 47), - BPF_ALU64_REG(BPF_ARSH, R0, R3), - BPF_JMP_IMM(BPF_JEQ, R0, -617, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R2, 1), - BPF_ALU64_REG(BPF_LSH, R4, R2), /* R4 = 46 << 1 */ - BPF_JMP_IMM(BPF_JEQ, R4, 92, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R4, 4), - BPF_ALU64_REG(BPF_LSH, R4, R4), /* R4 = 4 << 4 */ - BPF_JMP_IMM(BPF_JEQ, R4, 64, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R4, 5), - BPF_ALU32_REG(BPF_LSH, R4, R4), /* R4 = 5 << 5 */ - BPF_JMP_IMM(BPF_JEQ, R4, 160, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R0, -1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, -#ifdef CONFIG_32BIT - { - "INT: 32-bit context pointer word order and zero-extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP32_IMM(BPF_JEQ, R1, 0, 3), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_JMP32_IMM(BPF_JNE, R1, 0, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, -#endif - { - "check: missing ret", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: div_k_0", - .u.insns = { - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: unknown insn", - .u.insns = { - /* seccomp insn, rejected in socket filter */ - BPF_STMT(BPF_LDX | BPF_W | BPF_ABS, 0), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: out of range spill/fill", - .u.insns = { - BPF_STMT(BPF_STX, 16), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "JUMPS + HOLES", - .u.insns = { - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 15), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90c2894d, 3, 4), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90c2894d, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 14, 15), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x2ac28349, 2, 3), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x2ac28349, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 14, 15), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90d2ff41, 2, 3), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90d2ff41, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC, - { 0x00, 0x1b, 0x21, 0x3c, 0x9d, 0xf8, - 0x90, 0xe2, 0xba, 0x0a, 0x56, 0xb4, - 0x08, 0x00, - 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, - 0x20, 0x00, 0x40, 0x11, 0x00, 0x00, /* IP header */ - 0xc0, 0xa8, 0x33, 0x01, - 0xc0, 0xa8, 0x33, 0x02, - 0xbb, 0xb6, - 0xa9, 0xfa, - 0x00, 0x14, 0x00, 0x00, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc }, - { { 88, 0x001b } } - }, - { - "check: RET X", - .u.insns = { - BPF_STMT(BPF_RET | BPF_X, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: LDX + RET X", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 42), - BPF_STMT(BPF_RET | BPF_X, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { /* Mainly checking JIT here. */ - "M[]: alt STX + LDX", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 100), - BPF_STMT(BPF_STX, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 1), - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 2), - BPF_STMT(BPF_LDX | BPF_MEM, 2), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 3), - BPF_STMT(BPF_LDX | BPF_MEM, 3), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 4), - BPF_STMT(BPF_LDX | BPF_MEM, 4), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 5), - BPF_STMT(BPF_LDX | BPF_MEM, 5), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 6), - BPF_STMT(BPF_LDX | BPF_MEM, 6), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 7), - BPF_STMT(BPF_LDX | BPF_MEM, 7), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 8), - BPF_STMT(BPF_LDX | BPF_MEM, 8), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 9), - BPF_STMT(BPF_LDX | BPF_MEM, 9), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 10), - BPF_STMT(BPF_LDX | BPF_MEM, 10), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 11), - BPF_STMT(BPF_LDX | BPF_MEM, 11), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 12), - BPF_STMT(BPF_LDX | BPF_MEM, 12), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 13), - BPF_STMT(BPF_LDX | BPF_MEM, 13), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 14), - BPF_STMT(BPF_LDX | BPF_MEM, 14), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 15), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 116 } }, - }, - { /* Mainly checking JIT here. */ - "M[]: full STX + full LDX", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0xbadfeedb), - BPF_STMT(BPF_STX, 0), - BPF_STMT(BPF_LDX | BPF_IMM, 0xecabedae), - BPF_STMT(BPF_STX, 1), - BPF_STMT(BPF_LDX | BPF_IMM, 0xafccfeaf), - BPF_STMT(BPF_STX, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbffdcedc), - BPF_STMT(BPF_STX, 3), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfbbbdccb), - BPF_STMT(BPF_STX, 4), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfbabcbda), - BPF_STMT(BPF_STX, 5), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaedecbdb), - BPF_STMT(BPF_STX, 6), - BPF_STMT(BPF_LDX | BPF_IMM, 0xadebbade), - BPF_STMT(BPF_STX, 7), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfcfcfaec), - BPF_STMT(BPF_STX, 8), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbcdddbdc), - BPF_STMT(BPF_STX, 9), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfeefdfac), - BPF_STMT(BPF_STX, 10), - BPF_STMT(BPF_LDX | BPF_IMM, 0xcddcdeea), - BPF_STMT(BPF_STX, 11), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaccfaebb), - BPF_STMT(BPF_STX, 12), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbdcccdcf), - BPF_STMT(BPF_STX, 13), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaaedecde), - BPF_STMT(BPF_STX, 14), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfaeacdad), - BPF_STMT(BPF_STX, 15), - BPF_STMT(BPF_LDX | BPF_MEM, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 2), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 3), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 4), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 5), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 6), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 7), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 8), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 9), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 10), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 11), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 12), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 13), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 14), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x2a5a5e5 } }, - }, - { - "check: SKF_AD_MAX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_MAX), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { /* Passes checker but fails during runtime. */ - "LD [SKF_AD_OFF-1]", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF - 1), - BPF_STMT(BPF_RET | BPF_K, 1), - }, - CLASSIC, - { }, - { { 1, 0 } }, - }, - { - "load 64-bit immediate", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x567800001234LL), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_IMM(BPF_LSH, R3, 32), - BPF_ALU64_IMM(BPF_RSH, R3, 32), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R2, 0x5678, 1), - BPF_EXIT_INSN(), - BPF_JMP_IMM(BPF_JEQ, R3, 0x1234, 1), - BPF_EXIT_INSN(), - BPF_LD_IMM64(R0, 0x1ffffffffLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), /* R0 = 1 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_MOV | BPF_X */ - { - "ALU_MOV_X: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_MOV_X: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU32_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU64_MOV_X: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOV_X: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - /* BPF_ALU | BPF_MOV | BPF_K */ - { - "ALU_MOV_K: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_MOV_K: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_MOV_K: 0x0000ffffffff0000 = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x00000000ffffffffLL), - BPF_ALU32_IMM(BPF_MOV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_MOV_K: small negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "ALU_MOV_K: small negative zero extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU_MOV_K: large negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123456789), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123456789 } } - }, - { - "ALU_MOV_K: large negative zero extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123456789), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_MOV_K: dst = 2", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOV_K: dst = 2147483647", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_OR_K: dst = 0x0", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0), - BPF_ALU64_IMM(BPF_MOV, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOV_K: dst = -1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_MOV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOV_K: small negative", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "ALU64_MOV_K: small negative sign extension", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } } - }, - { - "ALU64_MOV_K: large negative", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123456789), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123456789 } } - }, - { - "ALU64_MOV_K: large negative sign extension", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123456789), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } } - }, - /* MOVSX32 */ - { - "ALU_MOVSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x00000000ffffffefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX32_REG(R1, R3, 8), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_MOVSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x00000000ffffbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX32_REG(R1, R3, 16), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* MOVSX64 REG */ - { - "ALU64_MOVSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 8), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOVSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 16), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOVSX | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffdeadbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 32), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_ADD | BPF_X */ - { - "ALU_ADD_X: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_X: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_ADD_X: 2 + 4294967294 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_LD_IMM64(R1, 4294967294U), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_ADD_X: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_X: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU64_ADD_X: 2 + 4294967294 = 4294967296", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_LD_IMM64(R1, 4294967294U), - BPF_LD_IMM64(R2, 4294967296ULL), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_JMP_REG(BPF_JEQ, R0, R2, 2), - BPF_MOV32_IMM(R0, 0), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_ALU | BPF_ADD | BPF_K */ - { - "ALU_ADD_K: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_K: 3 + 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_ADD, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_K: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 4294967294U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_ADD_K: 4294967294 + 2 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_ALU32_IMM(BPF_ADD, R0, 2), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_ADD_K: 0 + (-1) = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x00000000ffffffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0xffff = 0xffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0xffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x7fffffff = 0x7fffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x7fffffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0x7fffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80000000 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x80000000), - BPF_ALU32_IMM(BPF_ADD, R2, 0x80000000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80008000 = 0x80008000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x80008000), - BPF_ALU32_IMM(BPF_ADD, R2, 0x80008000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_ADD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_K: 3 + 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_ADD, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_K: 1 + 2147483646 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_ADD, R0, 2147483646), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_ADD_K: 4294967294 + 2 = 4294967296", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_LD_IMM64(R1, 4294967296ULL), - BPF_ALU64_IMM(BPF_ADD, R0, 2), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_ADD_K: 2147483646 + -2147483647 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483646), - BPF_ALU64_IMM(BPF_ADD, R0, -2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - { - "ALU64_ADD_K: 1 + 0 = 1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0x1), - BPF_ALU64_IMM(BPF_ADD, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + (-1) = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_ADD, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0xffff = 0xffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffff), - BPF_ALU64_IMM(BPF_ADD, R2, 0xffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0x7fffffff = 0x7fffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x7fffffff), - BPF_ALU64_IMM(BPF_ADD, R2, 0x7fffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0x80000000 = 0xffffffff80000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffff80000000LL), - BPF_ALU64_IMM(BPF_ADD, R2, 0x80000000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80008000 = 0xffffffff80008000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffff80008000LL), - BPF_ALU64_IMM(BPF_ADD, R2, 0x80008000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_SUB | BPF_X */ - { - "ALU_SUB_X: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_SUB_X: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU32_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_SUB_X: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_SUB_X: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_ALU | BPF_SUB | BPF_K */ - { - "ALU_SUB_K: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_SUB, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_SUB_K: 3 - 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_SUB, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_SUB_K: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_SUB, R0, 4294967294U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_SUB_K: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_SUB, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_SUB_K: 3 - 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_SUB, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_SUB_K: 4294967294 - 4294967295 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_ALU64_IMM(BPF_SUB, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - { - "ALU64_ADD_K: 2147483646 - 2147483647 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483646), - BPF_ALU64_IMM(BPF_SUB, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_MUL | BPF_X */ - { - "ALU_MUL_X: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 3), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU_MUL_X: 2 * 0x7FFFFFF8 = 0xFFFFFFF0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0x7FFFFFF8), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xFFFFFFF0 } }, - }, - { - "ALU_MUL_X: -1 * -1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, -1), - BPF_ALU32_IMM(BPF_MOV, R1, -1), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MUL_X: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 3), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU64_MUL_X: 1 * 2147483647 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483647), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_MUL_X: 64x64 multiply, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0fedcba987654321LL), - BPF_LD_IMM64(R1, 0x123456789abcdef0LL), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xe5618cf0 } } - }, - { - "ALU64_MUL_X: 64x64 multiply, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0fedcba987654321LL), - BPF_LD_IMM64(R1, 0x123456789abcdef0LL), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x2236d88f } } - }, - /* BPF_ALU | BPF_MUL | BPF_K */ - { - "ALU_MUL_K: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MUL, R0, 3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU_MUL_K: 3 * 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MUL, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_MUL_K: 2 * 0x7FFFFFF8 = 0xFFFFFFF0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MUL, R0, 0x7FFFFFF8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xFFFFFFF0 } }, - }, - { - "ALU_MUL_K: 1 * (-1) = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0x00000000ffffffff), - BPF_ALU32_IMM(BPF_MUL, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MUL_K: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU64_IMM(BPF_MUL, R0, 3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU64_MUL_K: 3 * 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MUL, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_MUL_K: 1 * 2147483647 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_MUL, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_MUL_K: 1 * -2147483647 = -2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_MUL, R0, -2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -2147483647 } }, - }, - { - "ALU64_MUL_K: 1 * (-1) = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_MUL, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MUL_K: 64x32 multiply, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xe242d208 } } - }, - { - "ALU64_MUL_K: 64x32 multiply, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xc28f5c28 } } - }, - /* BPF_ALU | BPF_DIV | BPF_X */ - { - "ALU_DIV_X: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_X: 4294967295 / 4294967295 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU32_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_X: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_X: 2147483647 / 2147483647 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483647), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_X: 0xffffffffffffffff / (-1) = 0x0000000000000001", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R4, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x0000000000000001LL), - BPF_ALU64_REG(BPF_DIV, R2, R4), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_K */ - { - "ALU_DIV_K: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_DIV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_K: 3 / 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_DIV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_K: 4294967295 / 4294967295 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_DIV, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_DIV_K: 0xffffffffffffffff / (-1) = 0x1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x1UL), - BPF_ALU32_IMM(BPF_DIV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_DIV_K: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU64_IMM(BPF_DIV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_K: 3 / 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_DIV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_K: 2147483647 / 2147483647 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU64_IMM(BPF_DIV, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_K: 0xffffffffffffffff / (-1) = 0x0000000000000001", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x0000000000000001LL), - BPF_ALU64_IMM(BPF_DIV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_X */ - { - "ALU_MOD_X: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_MOD_X: 4294967295 % 4294967293 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967293U), - BPF_ALU32_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOD_X: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MOD_X: 2147483647 % 2147483645 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483645), - BPF_ALU64_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_K */ - { - "ALU_MOD_K: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_MOD_K: 3 % 1 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOD, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - }, - { - "ALU_MOD_K: 4294967295 % 4294967293 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOD, R0, 4294967293U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOD_K: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MOD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MOD_K: 3 % 1 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MOD, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - }, - { - "ALU64_MOD_K: 2147483647 % 2147483645 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU64_IMM(BPF_MOD, R0, 2147483645), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_X off=1 (SDIV) */ - { - "ALU_SDIV_X: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG_OFF(BPF_DIV, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_K off=1 (SDIV) */ - { - "ALU_SDIV_K: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM_OFF(BPF_DIV, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU64 | BPF_DIV | BPF_X off=1 (SDIV64) */ - { - "ALU64_SDIV_X: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG_OFF(BPF_DIV, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU64 | BPF_DIV | BPF_K off=1 (SDIV64) */ - { - "ALU64_SDIV_K: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU64_IMM_OFF(BPF_DIV, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_X off=1 (SMOD) */ - { - "ALU_SMOD_X: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG_OFF(BPF_MOD, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_K off=1 (SMOD) */ - { - "ALU_SMOD_K: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM_OFF(BPF_MOD, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU64 | BPF_MOD | BPF_X off=1 (SMOD64) */ - { - "ALU64_SMOD_X: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG_OFF(BPF_MOD, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU64 | BPF_MOD | BPF_K off=1 (SMOD64) */ - { - "ALU64_SMOD_K: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU64_IMM_OFF(BPF_MOD, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_AND | BPF_X */ - { - "ALU_AND_X: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_AND_X: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_AND_X: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_AND_X: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - /* BPF_ALU | BPF_AND | BPF_K */ - { - "ALU_AND_K: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_AND, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_AND_K: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_AND, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU_AND_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_AND, R0, 15), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4 } } - }, - { - "ALU_AND_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), - BPF_ALU32_IMM(BPF_AND, R0, 0xafbfcfdf), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xa1b2c3d4 } } - }, - { - "ALU_AND_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0000000080a0c0e0LL), - BPF_ALU32_IMM(BPF_AND, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_AND_K: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_AND, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_AND_K: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU64_IMM(BPF_AND, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000000000000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000000000000000LL), - BPF_ALU64_IMM(BPF_AND, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: 0xffffffffffffffff & -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000090b0d0fLL), - BPF_ALU64_IMM(BPF_AND, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_AND_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0123456780a0c0e0LL), - BPF_ALU64_IMM(BPF_AND, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_OR | BPF_X */ - { - "ALU_OR_X: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_OR_X: 0x0 | 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_OR_X: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_OR_X: 0 | 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - /* BPF_ALU | BPF_OR | BPF_K */ - { - "ALU_OR_K: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_OR, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_OR_K: 0 & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_OR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU_OR_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_OR, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01020305 } } - }, - { - "ALU_OR_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_OR, R0, 0xa0b0c0d0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xa1b2c3d4 } } - }, - { - "ALU_OR_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000f9fbfdffLL), - BPF_ALU32_IMM(BPF_OR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_OR_K: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_OR, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_OR_K: 0 & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU64_IMM(BPF_OR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_OR, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: 0x0000ffffffff0000 | -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: 0x000000000000000 | -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000000000000000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x012345678fafcfefLL), - BPF_ALU64_IMM(BPF_OR, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_OR_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0xfffffffff9fbfdffLL), - BPF_ALU64_IMM(BPF_OR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_XOR | BPF_X */ - { - "ALU_XOR_X: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_MOV, R1, 6), - BPF_ALU32_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_XOR_X: 0x1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU64_XOR_X: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_MOV, R1, 6), - BPF_ALU64_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_XOR_X: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - /* BPF_ALU | BPF_XOR | BPF_K */ - { - "ALU_XOR_K: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_XOR, R0, 6), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_XOR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU_XOR_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_XOR, R0, 15), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x0102030b } } - }, - { - "ALU_XOR_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), - BPF_ALU32_IMM(BPF_XOR, R0, 0xafbfcfdf), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x5e4d3c2b } } - }, - { - "ALU_XOR_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000795b3d1fLL), - BPF_ALU32_IMM(BPF_XOR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_XOR_K: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU64_IMM(BPF_XOR, R0, 6), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_XOR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU64_XOR_K: 0x0000ffffffff0000 ^ 0x0 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_XOR, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: 0x0000ffffffff0000 ^ -1 = 0xffff00000000ffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffff00000000ffffLL), - BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: 0x000000000000000 ^ -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000000000000000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0123456786a4c2e0LL), - BPF_ALU64_IMM(BPF_XOR, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_XOR_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0xfedcba98795b3d1fLL), - BPF_ALU64_IMM(BPF_XOR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_LSH | BPF_X */ - { - "ALU_LSH_X: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_LSH_X: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU_LSH_X: 0x12345678 << 12 = 0x45678000", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x45678000 } } - }, - { - "ALU64_LSH_X: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_LSH_X: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU64_LSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xbcdef000 } } - }, - { - "ALU64_LSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x3456789a } } - }, - { - "ALU64_LSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x9abcdef0 } } - }, - { - "ALU64_LSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } } - }, - /* BPF_ALU | BPF_LSH | BPF_K */ - { - "ALU_LSH_K: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_LSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_LSH_K: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_LSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU_LSH_K: 0x12345678 << 12 = 0x45678000", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_LSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x45678000 } } - }, - { - "ALU_LSH_K: 0x12345678 << 0 = 0x12345678", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_LSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x12345678 } } - }, - { - "ALU64_LSH_K: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_LSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_LSH_K: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_LSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU64_LSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xbcdef000 } } - }, - { - "ALU64_LSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x3456789a } } - }, - { - "ALU64_LSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x9abcdef0 } } - }, - { - "ALU64_LSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_RSH | BPF_X */ - { - "ALU_RSH_X: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_X: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_X: 0x12345678 >> 20 = 0x123", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_MOV, R1, 20), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x123 } } - }, - { - "ALU64_RSH_X: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_X: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_RSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x00081234 } } - }, - { - "ALU64_RSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x08123456 } } - }, - { - "ALU64_RSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_RSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_RSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - /* BPF_ALU | BPF_RSH | BPF_K */ - { - "ALU_RSH_K: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_RSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_K: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_RSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_K: 0x12345678 >> 20 = 0x123", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_RSH, R0, 20), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x123 } } - }, - { - "ALU_RSH_K: 0x12345678 >> 0 = 0x12345678", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_RSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x12345678 } } - }, - { - "ALU64_RSH_K: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU64_IMM(BPF_RSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_K: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU64_IMM(BPF_RSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_RSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x00081234 } } - }, - { - "ALU64_RSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x08123456 } } - }, - { - "ALU64_RSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_RSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_ARSH | BPF_X */ - { - "ALU32_ARSH_X: -1234 >> 7 = -10", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_MOV, R1, 7), - BPF_ALU32_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -10 } } - }, - { - "ALU64_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xff00ff0000000000LL), - BPF_ALU32_IMM(BPF_MOV, R1, 40), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff00ff } }, - }, - { - "ALU64_ARSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_ARSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfff81234 } } - }, - { - "ALU64_ARSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xf8123456 } } - }, - { - "ALU64_ARSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_ARSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_ARSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - /* BPF_ALU | BPF_ARSH | BPF_K */ - { - "ALU32_ARSH_K: -1234 >> 7 = -10", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_ARSH, R0, 7), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -10 } } - }, - { - "ALU32_ARSH_K: -1234 >> 0 = -1234", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_ARSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1234 } } - }, - { - "ALU64_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xff00ff0000000000LL), - BPF_ALU64_IMM(BPF_ARSH, R0, 40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff00ff } }, - }, - { - "ALU64_ARSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_ARSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfff81234 } } - }, - { - "ALU64_ARSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xf8123456 } } - }, - { - "ALU64_ARSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xf123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_ARSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_NEG */ - { - "ALU_NEG: -(3) = -3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 3), - BPF_ALU32_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - { - "ALU_NEG: -(-3) = 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -3), - BPF_ALU32_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_NEG: -(3) = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - { - "ALU64_NEG: -(-3) = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, -3), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_BE */ - { - "ALU_END_FROM_BE 16: 0x0123456789abcdef -> 0xcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be16(0xcdef) } }, - }, - { - "ALU_END_FROM_BE 32: 0x0123456789abcdef -> 0x89abcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be32(0x89abcdef) } }, - }, - { - "ALU_END_FROM_BE 64: 0x0123456789abcdef -> 0x89abcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } }, - }, - { - "ALU_END_FROM_BE 64: 0x0123456789abcdef >> 32 -> 0x01234567", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_be64(0x0123456789abcdefLL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_BE, reversed */ - { - "ALU_END_FROM_BE 16: 0xfedcba9876543210 -> 0x3210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be16(0x3210) } }, - }, - { - "ALU_END_FROM_BE 32: 0xfedcba9876543210 -> 0x76543210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be32(0x76543210) } }, - }, - { - "ALU_END_FROM_BE 64: 0xfedcba9876543210 -> 0x76543210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_be64(0xfedcba9876543210ULL) } }, - }, - { - "ALU_END_FROM_BE 64: 0xfedcba9876543210 >> 32 -> 0xfedcba98", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_be64(0xfedcba9876543210ULL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_LE */ - { - "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le16(0xcdef) } }, - }, - { - "ALU_END_FROM_LE 32: 0x0123456789abcdef -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le32(0x89abcdef) } }, - }, - { - "ALU_END_FROM_LE 64: 0x0123456789abcdef -> 0x67452301", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } }, - }, - { - "ALU_END_FROM_LE 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_le64(0x0123456789abcdefLL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_LE, reversed */ - { - "ALU_END_FROM_LE 16: 0xfedcba9876543210 -> 0x1032", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le16(0x3210) } }, - }, - { - "ALU_END_FROM_LE 32: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le32(0x76543210) } }, - }, - { - "ALU_END_FROM_LE 64: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_le64(0xfedcba9876543210ULL) } }, - }, - { - "ALU_END_FROM_LE 64: 0xfedcba9876543210 >> 32 -> 0x98badcfe", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, - }, - /* BSWAP */ - { - "BSWAP 16: 0x0123456789abcdef -> 0xefcd", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcd } }, - }, - { - "BSWAP 32: 0x0123456789abcdef -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcdab89 } }, - }, - { - "BSWAP 64: 0x0123456789abcdef -> 0x67452301", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x67452301 } }, - }, - { - "BSWAP 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcdab89 } }, - }, - /* BSWAP, reversed */ - { - "BSWAP 16: 0xfedcba9876543210 -> 0x1032", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1032 } }, - }, - { - "BSWAP 32: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x10325476 } }, - }, - { - "BSWAP 64: 0xfedcba9876543210 -> 0x98badcfe", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x98badcfe } }, - }, - { - "BSWAP 64: 0xfedcba9876543210 >> 32 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x10325476 } }, - }, - /* BPF_LDX_MEM B/H/W/DW */ - { - "BPF_LDX_MEM | BPF_B, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000000000008ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEM(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_B, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEM(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_B, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_B, R1, R2, -256), - BPF_LDX_MEM(BPF_B, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_B, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_B, R1, R2, 256), - BPF_LDX_MEM(BPF_B, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_B, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_B, R1, R2, 4096), - BPF_LDX_MEM(BPF_B, R0, R1, 4096), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 4096 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000000000708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEM(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_H, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEM(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_H, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_H, R1, R2, -256), - BPF_LDX_MEM(BPF_H, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 256), - BPF_LDX_MEM(BPF_H, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 8192), - BPF_LDX_MEM(BPF_H, R0, R1, 8192), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 8192 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 13), - BPF_LDX_MEM(BPF_H, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000005060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEM(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_W, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEM(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_W, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_W, R1, R2, -256), - BPF_LDX_MEM(BPF_W, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 256), - BPF_LDX_MEM(BPF_W, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 16384), - BPF_LDX_MEM(BPF_W, R0, R1, 16384), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 16384 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 13), - BPF_LDX_MEM(BPF_W, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_DW, R1, R2, -256), - BPF_LDX_MEM(BPF_DW, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 256), - BPF_LDX_MEM(BPF_DW, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 32760), - BPF_LDX_MEM(BPF_DW, R0, R1, 32760), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32768, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 13), - BPF_LDX_MEM(BPF_DW, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - /* BPF_LDX_MEMSX B/H/W */ - { - "BPF_LDX_MEMSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xdead0000000000f0ULL), - BPF_LD_IMM64(R2, 0xfffffffffffffff0ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEMSX(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEMSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xdead00000000f123ULL), - BPF_LD_IMM64(R2, 0xfffffffffffff123ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEMSX(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEMSX | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x00000000deadbeefULL), - BPF_LD_IMM64(R2, 0xffffffffdeadbeefULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEMSX(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* BPF_STX_MEM B/H/W/DW */ - { - "BPF_STX_MEM | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d0e008ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_B, R10, R2, -1), -#else - BPF_STX_MEM(BPF_B, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_B, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d0e088ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_B, R10, R2, -1), -#else - BPF_STX_MEM(BPF_B, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d00708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_H, R10, R2, -2), -#else - BPF_STX_MEM(BPF_H, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_H, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d08788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_H, R10, R2, -2), -#else - BPF_STX_MEM(BPF_H, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b005060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_W, R10, R2, -4), -#else - BPF_STX_MEM(BPF_W, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_W, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b085868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_W, R10, R2, -4), -#else - BPF_STX_MEM(BPF_W, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ - { - "ST_MEM_B: Store/Load byte: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_B, R10, -40, 0xff), - BPF_LDX_MEM(BPF_B, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xff } }, - .stack_depth = 40, - }, - { - "ST_MEM_B: Store/Load byte: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0x7f), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7f } }, - .stack_depth = 40, - }, - { - "STX_MEM_B: Store/Load byte: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffLL), - BPF_STX_MEM(BPF_B, R10, R1, -40), - BPF_LDX_MEM(BPF_B, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xff } }, - .stack_depth = 40, - }, - { - "ST_MEM_H: Store/Load half word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0xffff), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_H: Store/Load half word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0x7fff), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fff } }, - .stack_depth = 40, - }, - { - "STX_MEM_H: Store/Load half word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffLL), - BPF_STX_MEM(BPF_H, R10, R1, -40), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_W: Store/Load word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_W, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_W: Store/Load word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_W, R10, -40, 0x7fffffff), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_W: Store/Load word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffffffLL), - BPF_STX_MEM(BPF_W, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max negative 2", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffff00000000ffffLL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_DW, R2, R10, -40), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_DW, R10, -40, 0x7fffffff), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store/Load double word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store double word: first word in memory", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0x0123456789abcdefLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, -#ifdef __BIG_ENDIAN - { { 0, 0x01234567 } }, -#else - { { 0, 0x89abcdef } }, -#endif - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store double word: second word in memory", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0x0123456789abcdefLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, -#ifdef __BIG_ENDIAN - { { 0, 0x89abcdef } }, -#else - { { 0, 0x01234567 } }, -#endif - .stack_depth = 40, - }, - /* BPF_STX | BPF_ATOMIC | BPF_W/DW */ - { - "STX_XADD_W: X + 1 + 1 + 1 + ...", - { }, - INTERNAL, - { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxw, - }, - { - "STX_XADD_DW: X + 1 + 1 + 1 + ...", - { }, - INTERNAL, - { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxdw, - }, - /* - * Exhaustive tests of atomic operation variants. - * Individual tests are expanded from template macros for all - * combinations of ALU operation, word size and fetching. - */ -#define BPF_ATOMIC_POISON(width) ((width) == BPF_W ? (0xbaadf00dULL << 32) : 0) - -#define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_LD_IMM64(R5, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R5, -40), \ - BPF_LDX_MEM(width, R0, R10, -40), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, result } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST2(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test side effects, r10: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_ALU64_REG(BPF_MOV, R1, R10), \ - BPF_LD_IMM64(R0, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(BPF_W, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R0, -40), \ - BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST3(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test side effects, r0: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_LD_IMM64(R1, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R1, -40), \ - BPF_ALU64_REG(BPF_SUB, R0, R10), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST4(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test fetch: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_LD_IMM64(R3, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R3, -40), \ - BPF_ALU32_REG(BPF_MOV, R0, R3), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, (op) & BPF_FETCH ? old : update } }, \ - .stack_depth = 40, \ -} - /* BPF_ATOMIC | BPF_W: BPF_ADD */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_W: BPF_ADD | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_DW: BPF_ADD */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_DW: BPF_ADD | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_W: BPF_AND */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_W: BPF_AND | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_DW: BPF_AND */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_DW: BPF_AND | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_W: BPF_OR */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_W: BPF_OR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_DW: BPF_OR */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_DW: BPF_OR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_W: BPF_XOR */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_W: BPF_XOR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_DW: BPF_XOR */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_DW: BPF_XOR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_W: BPF_XCHG */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - /* BPF_ATOMIC | BPF_DW: BPF_XCHG */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), -#undef BPF_ATOMIC_POISON -#undef BPF_ATOMIC_OP_TEST1 -#undef BPF_ATOMIC_OP_TEST2 -#undef BPF_ATOMIC_OP_TEST3 -#undef BPF_ATOMIC_OP_TEST4 - /* BPF_ATOMIC | BPF_W, BPF_CMPXCHG */ - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful return", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful store", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure return", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure store", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test side effects", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_ALU32_REG(BPF_MOV, R0, R3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } }, - .stack_depth = 40, - }, - /* BPF_ATOMIC | BPF_DW, BPF_CMPXCHG */ - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful return", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful store", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R0, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure return", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure store", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test side effects", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - /* BPF_JMP32 | BPF_JEQ | BPF_K */ - { - "JMP32_JEQ_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JEQ, R0, 321, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JEQ_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 12345678), - BPF_JMP32_IMM(BPF_JEQ, R0, 12345678 & 0xffff, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, 12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345678 } } - }, - { - "JMP32_JEQ_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JEQ | BPF_X */ - { - "JMP32_JEQ_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1234), - BPF_ALU32_IMM(BPF_MOV, R1, 4321), - BPF_JMP32_REG(BPF_JEQ, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1234), - BPF_JMP32_REG(BPF_JEQ, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1234 } } - }, - /* BPF_JMP32 | BPF_JNE | BPF_K */ - { - "JMP32_JNE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 321, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JNE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 12345678), - BPF_JMP32_IMM(BPF_JNE, R0, 12345678, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 12345678 & 0xffff, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345678 } } - }, - { - "JMP32_JNE_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JNE, R0, -123, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JNE | BPF_X */ - { - "JMP32_JNE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1234), - BPF_ALU32_IMM(BPF_MOV, R1, 1234), - BPF_JMP32_REG(BPF_JNE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 4321), - BPF_JMP32_REG(BPF_JNE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1234 } } - }, - /* BPF_JMP32 | BPF_JSET | BPF_K */ - { - "JMP32_JSET_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 2, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 3, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "JMP32_JSET_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x40000000), - BPF_JMP32_IMM(BPF_JSET, R0, 0x3fffffff, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 0x60000000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x40000000 } } - }, - { - "JMP32_JSET_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSET, R0, -1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JSET | BPF_X */ - { - "JMP32_JSET_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 8), - BPF_ALU32_IMM(BPF_MOV, R1, 7), - BPF_JMP32_REG(BPF_JSET, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 8 | 2), - BPF_JMP32_REG(BPF_JNE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 8 } } - }, - /* BPF_JMP32 | BPF_JGT | BPF_K */ - { - "JMP32_JGT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JGT, R0, 123, 1), - BPF_JMP32_IMM(BPF_JGT, R0, 122, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JGT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JGT, R0, 0xffffffff, 1), - BPF_JMP32_IMM(BPF_JGT, R0, 0xfffffffd, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGT | BPF_X */ - { - "JMP32_JGT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JGT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JGT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGE | BPF_K */ - { - "JMP32_JGE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JGE, R0, 124, 1), - BPF_JMP32_IMM(BPF_JGE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JGE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JGE, R0, 0xffffffff, 1), - BPF_JMP32_IMM(BPF_JGE, R0, 0xfffffffe, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGE | BPF_X */ - { - "JMP32_JGE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JGE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), - BPF_JMP32_REG(BPF_JGE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLT | BPF_K */ - { - "JMP32_JLT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JLT, R0, 123, 1), - BPF_JMP32_IMM(BPF_JLT, R0, 124, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JLT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JLT, R0, 0xfffffffd, 1), - BPF_JMP32_IMM(BPF_JLT, R0, 0xffffffff, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLT | BPF_X */ - { - "JMP32_JLT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JLT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JLT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLE | BPF_K */ - { - "JMP32_JLE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JLE, R0, 122, 1), - BPF_JMP32_IMM(BPF_JLE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JLE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffd, 1), - BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffe, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLE | BPF_X */ - { - "JMP32_JLE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JLE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), - BPF_JMP32_REG(BPF_JLE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JSGT | BPF_K */ - { - "JMP32_JSGT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSGT, R0, -123, 1), - BPF_JMP32_IMM(BPF_JSGT, R0, -124, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSGT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSGT, R0, -12345678, 1), - BPF_JMP32_IMM(BPF_JSGT, R0, -12345679, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGT | BPF_X */ - { - "JMP32_JSGT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSGT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345679), - BPF_JMP32_REG(BPF_JSGT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGE | BPF_K */ - { - "JMP32_JSGE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSGE, R0, -122, 1), - BPF_JMP32_IMM(BPF_JSGE, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSGE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSGE, R0, -12345677, 1), - BPF_JMP32_IMM(BPF_JSGE, R0, -12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGE | BPF_X */ - { - "JMP32_JSGE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345677), - BPF_JMP32_REG(BPF_JSGE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSGE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLT | BPF_K */ - { - "JMP32_JSLT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSLT, R0, -123, 1), - BPF_JMP32_IMM(BPF_JSLT, R0, -122, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSLT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSLT, R0, -12345678, 1), - BPF_JMP32_IMM(BPF_JSLT, R0, -12345677, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLT | BPF_X */ - { - "JMP32_JSLT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSLT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345677), - BPF_JMP32_REG(BPF_JSLT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLE | BPF_K */ - { - "JMP32_JSLE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSLE, R0, -124, 1), - BPF_JMP32_IMM(BPF_JSLE, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSLE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSLE, R0, -12345679, 1), - BPF_JMP32_IMM(BPF_JSLE, R0, -12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLE | BPF_K */ - { - "JMP32_JSLE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345679), - BPF_JMP32_REG(BPF_JSLE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSLE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP | BPF_EXIT */ - { - "JMP_EXIT", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x4711), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0x4712), - }, - INTERNAL, - { }, - { { 0, 0x4711 } }, - }, - /* BPF_JMP | BPF_JA */ - { - "JMP_JA: Unconditional jump: if (true) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JA, 0, 0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP32 | BPF_JA */ - { - "JMP32_JA: Unconditional jump: if (true) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP32_IMM(BPF_JA, 0, 1, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLT | BPF_K */ - { - "JMP_JSLT_K: Signed jump: if (-2 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xfffffffffffffffeLL), - BPF_JMP_IMM(BPF_JSLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLT_K: Signed jump: if (-1 < -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGT | BPF_K */ - { - "JMP_JSGT_K: Signed jump: if (-1 > -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGT, R1, -2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGT_K: Signed jump: if (-1 > -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLE | BPF_K */ - { - "JMP_JSLE_K: Signed jump: if (-2 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xfffffffffffffffeLL), - BPF_JMP_IMM(BPF_JSLE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: if (-1 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSLE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: value walk 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 6), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 4), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 2), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: value walk 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 4), - BPF_ALU64_IMM(BPF_SUB, R1, 2), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 2), - BPF_ALU64_IMM(BPF_SUB, R1, 2), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGE | BPF_K */ - { - "JMP_JSGE_K: Signed jump: if (-1 >= -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGE, R1, -2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: if (-1 >= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: value walk 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -3), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 6), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 4), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 2), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: value walk 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -3), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 4), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 2), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_K */ - { - "JMP_JGT_K: if (3 > 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGT, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: Unsigned jump: if (-1 > 1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_JMP_IMM(BPF_JGT, R1, 1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_K */ - { - "JMP_JLT_K: if (2 < 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 2), - BPF_JMP_IMM(BPF_JLT, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: Unsigned jump: if (1 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 1), - BPF_JMP_IMM(BPF_JLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGE | BPF_K */ - { - "JMP_JGE_K: if (3 >= 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGE, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLE | BPF_K */ - { - "JMP_JLE_K: if (2 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 2), - BPF_JMP_IMM(BPF_JLE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_K jump backwards */ - { - "JMP_JGT_K: if (3 > 2) return 1 (jump backwards)", - .u.insns_int = { - BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */ - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */ - BPF_LD_IMM64(R1, 3), /* note: this takes 2 insns */ - BPF_JMP_IMM(BPF_JGT, R1, 2, -6), /* goto out */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_K: if (3 >= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_K jump backwards */ - { - "JMP_JGT_K: if (2 < 3) return 1 (jump backwards)", - .u.insns_int = { - BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */ - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */ - BPF_LD_IMM64(R1, 2), /* note: this takes 2 insns */ - BPF_JMP_IMM(BPF_JLT, R1, 3, -6), /* goto out */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_K: if (3 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JLE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JNE | BPF_K */ - { - "JMP_JNE_K: if (3 != 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JNE, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JEQ | BPF_K */ - { - "JMP_JEQ_K: if (3 == 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JEQ, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSET | BPF_K */ - { - "JMP_JSET_K: if (0x3 & 0x2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSET, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSET_K: if (0x3 & 0xffffffff) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSET, R1, 0xffffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGT | BPF_X */ - { - "JMP_JSGT_X: Signed jump: if (-1 > -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGT_X: Signed jump: if (-1 > -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLT | BPF_X */ - { - "JMP_JSLT_X: Signed jump: if (-2 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLT_X: Signed jump: if (-1 < -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSLT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGE | BPF_X */ - { - "JMP_JSGE_X: Signed jump: if (-1 >= -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_X: Signed jump: if (-1 >= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLE | BPF_X */ - { - "JMP_JSLE_X: Signed jump: if (-2 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSLE, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_X: Signed jump: if (-1 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSLE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_X */ - { - "JMP_JGT_X: if (3 > 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_X: Unsigned jump: if (-1 > 1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, 1), - BPF_JMP_REG(BPF_JGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_X */ - { - "JMP_JLT_X: if (2 < 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLT_X: Unsigned jump: if (1 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, 1), - BPF_JMP_REG(BPF_JLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGE | BPF_X */ - { - "JMP_JGE_X: if (3 >= 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_X: if (3 >= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLE | BPF_X */ - { - "JMP_JLE_X: if (2 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: if (3 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JLE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - /* Mainly testing JIT + imm64 here. */ - "JMP_JGE_X: ldimm64 test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 2), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xeeeeeeeeU } }, - }, - { - "JMP_JGE_X: ldimm64 test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 0), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffffU } }, - }, - { - "JMP_JGE_X: ldimm64 test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 4), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: ldimm64 test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 2), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xeeeeeeeeU } }, - }, - { - "JMP_JLE_X: ldimm64 test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 0), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffffU } }, - }, - { - "JMP_JLE_X: ldimm64 test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 4), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JNE | BPF_X */ - { - "JMP_JNE_X: if (3 != 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JNE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JEQ | BPF_X */ - { - "JMP_JEQ_X: if (3 == 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JEQ, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSET | BPF_X */ - { - "JMP_JSET_X: if (0x3 & 0x2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JSET, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSET_X: if (0x3 & 0xffffffff) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 0xffffffff), - BPF_JMP_REG(BPF_JSET, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JA: Jump, gap, jump, ...", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xababcbac } }, - .fill_helper = bpf_fill_ja, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Maximum possible literals", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xffffffff } }, - .fill_helper = bpf_fill_maxinsns1, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Single literal", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xfefefefe } }, - .fill_helper = bpf_fill_maxinsns2, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Run/add until end", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x947bf368 } }, - .fill_helper = bpf_fill_maxinsns3, - }, - { - "BPF_MAXINSNS: Too many instructions", - { }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = bpf_fill_maxinsns4, - .expected_errcode = -EINVAL, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long jump", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xabababab } }, - .fill_helper = bpf_fill_maxinsns5, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Ctx heavy transformations", - { }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_PRESENT }, - { 10, SKB_VLAN_PRESENT } - }, - .fill_helper = bpf_fill_maxinsns6, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Call heavy transformations", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 1, 0 }, { 10, 0 } }, - .fill_helper = bpf_fill_maxinsns7, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Jump heavy test", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xffffffff } }, - .fill_helper = bpf_fill_maxinsns8, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long jump backwards", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0xcbababab } }, - .fill_helper = bpf_fill_maxinsns9, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Edge hopping nuthouse", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0xabababac } }, - .fill_helper = bpf_fill_maxinsns10, - }, - { - "BPF_MAXINSNS: Jump, gap, jump, ...", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xababcbac } }, - .fill_helper = bpf_fill_maxinsns11, - }, - { - "BPF_MAXINSNS: jump over MSH", - { }, - CLASSIC | FLAG_EXPECTED_FAIL, - { 0xfa, 0xfb, 0xfc, 0xfd, }, - { { 4, 0xabababab } }, - .fill_helper = bpf_fill_maxinsns12, - .expected_errcode = -EINVAL, - }, - { - "BPF_MAXINSNS: exec all MSH", - { }, - CLASSIC, - { 0xfa, 0xfb, 0xfc, 0xfd, }, - { { 4, 0xababab83 } }, - .fill_helper = bpf_fill_maxinsns13, - }, - { - "BPF_MAXINSNS: ld_abs+get_processor_id", - { }, - CLASSIC, - { }, - { { 1, 0xbee } }, - .fill_helper = bpf_fill_ld_abs_get_processor_id, - }, - /* - * LD_IND / LD_ABS on fragmented SKBs - */ - { - "LD_IND byte frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x42} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND halfword frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x4344} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND word frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x8), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x21071983} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND halfword mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x0519} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_IND word mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x25051982} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_ABS byte frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x40), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x42} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS halfword frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x44), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x4344} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS word frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x48), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x21071983} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS halfword mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x0519} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_ABS word mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x25051982} }, - .frag_data = { 0x19, 0x82 }, - }, - /* - * LD_IND / LD_ABS on non fragmented SKBs - */ - { - /* - * this tests that the JIT/interpreter correctly resets X - * before using it in an LD_IND instruction. - */ - "LD_IND byte default X", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x1] = 0x42 }, - { {0x40, 0x42 } }, - }, - { - "LD_IND byte positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 } }, - }, - { - "LD_IND byte negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x05 } }, - }, - { - "LD_IND byte positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xff } }, - }, - { - "LD_IND byte positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND byte negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_IND byte negative offset, multiple calls", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3b), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 1), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 2), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 3), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_IND halfword positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0xdd88 } }, - }, - { - "LD_IND halfword negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0xbb66 } }, - }, - { - "LD_IND halfword unaligned", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0x66cc } }, - }, - { - "LD_IND halfword positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3d), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffff } }, - }, - { - "LD_IND halfword positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND halfword negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_IND word positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xee99ffaa } }, - }, - { - "LD_IND word negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xaa55bb66 } }, - }, - { - "LD_IND word unaligned (addr & 3 == 2)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xbb66cc77 } }, - }, - { - "LD_IND word unaligned (addr & 3 == 1)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x55bb66cc } }, - }, - { - "LD_IND word unaligned (addr & 3 == 3)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x66cc77dd } }, - }, - { - "LD_IND word positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3b), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffffffff } }, - }, - { - "LD_IND word positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND word negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_ABS byte", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x20), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xcc } }, - }, - { - "LD_ABS byte positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xff } }, - }, - { - "LD_ABS byte positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS byte negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS byte negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_ABS byte negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS byte negative offset, multiple calls", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3d), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_ABS halfword", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x22), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xdd88 } }, - }, - { - "LD_ABS halfword unaligned", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x25), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x99ff } }, - }, - { - "LD_ABS halfword positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffff } }, - }, - { - "LD_ABS halfword positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS halfword negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS halfword negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x1982 }, }, - }, - { - "LD_ABS halfword negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS word", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x1c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xaa55bb66 } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 2)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x22), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xdd88ee99 } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 1)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x21), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x77dd88ee } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 3)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x23), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x88ee99ff } }, - }, - { - "LD_ABS word positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffffffff } }, - }, - { - "LD_ABS word positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS word negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS word negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x25051982 }, }, - }, - { - "LD_ABS word negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LDX_MSH standalone, preserved A", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0xffeebbaa }, }, - }, - { - "LDX_MSH standalone, preserved A 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0x175e9d63), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3d), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x175e9d63 }, }, - }, - { - "LDX_MSH standalone, test result 1", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x14 }, }, - }, - { - "LDX_MSH standalone, test result 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x24 }, }, - }, - { - "LDX_MSH standalone, negative offset", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, -1), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0 }, }, - }, - { - "LDX_MSH standalone, negative offset 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x24 }, }, - }, - { - "LDX_MSH standalone, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x40), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0 }, }, - }, - /* - * verify that the interpreter or JIT correctly sets A and X - * to 0. - */ - { - "ADD default X", - .u.insns = { - /* - * A = 0x42 - * A = A + X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "ADD default A", - .u.insns = { - /* - * A = A + 0x42 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "SUB default X", - .u.insns = { - /* - * A = 0x66 - * A = A - X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x66), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x66 } }, - }, - { - "SUB default A", - .u.insns = { - /* - * A = A - -0x66 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_SUB | BPF_K, -0x66), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x66 } }, - }, - { - "MUL default X", - .u.insns = { - /* - * A = 0x42 - * A = A * X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_MUL | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MUL default A", - .u.insns = { - /* - * A = A * 0x66 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_MUL | BPF_K, 0x66), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "DIV default X", - .u.insns = { - /* - * A = 0x42 - * A = A / X ; this halt the filter execution if X is 0 - * ret 0x42 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_K, 0x42), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "DIV default A", - .u.insns = { - /* - * A = A / 1 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MOD default X", - .u.insns = { - /* - * A = 0x42 - * A = A mod X ; this halt the filter execution if X is 0 - * ret 0x42 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_K, 0x42), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MOD default A", - .u.insns = { - /* - * A = A mod 1 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_MOD | BPF_K, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "JMP EQ default A", - .u.insns = { - /* - * cmp A, 0x0, 0, 1 - * ret 0x42 - * ret 0x66 - */ - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_K, 0x66), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "JMP EQ default X", - .u.insns = { - /* - * A = 0x0 - * cmp A, X, 0, 1 - * ret 0x42 - * ret 0x66 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0x0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_K, 0x66), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - /* Checking interpreter vs JIT wrt signed extended imms. */ - { - "JNE signed compare, test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R2, -17104896, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R2, 0xfefb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_ALU32_IMM(BPF_MOV, R4, 0xfefb0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JNE, R2, R4, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 4", - .u.insns_int = { - BPF_LD_IMM64(R1, -17104896), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, -17104896, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 5", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xfefb0000), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, 0xfefb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 6", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x7efb0000), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, 0x7efb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 7", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffff0000), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xfefbbc12), - BPF_STMT(BPF_ALU | BPF_AND | BPF_X, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0xfefb0000, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, 2), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { { 0, 2 } }, - }, - /* BPF_LDX_MEM with operand aliasing */ - { - "LDX_MEM_B: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_B, R10, -8, 123), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_B, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_H: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_H, R10, -8, 12345), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_H, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_W: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -8, 123456789), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_W, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123456789 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_DW: operand register aliasing", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x123456789abcdefULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_DW, R0, R0, -8), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_MOV64_REG(R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* - * Register (non-)clobbering tests for the case where a JIT implements - * complex ALU or ATOMIC operations via function calls. If so, the - * function call must be transparent to the eBPF registers. The JIT - * must therefore save and restore relevant registers across the call. - * The following tests check that the eBPF registers retain their - * values after such an operation. Mainly intended for complex ALU - * and atomic operation, but we run it for all. You never know... - * - * Note that each operations should be tested twice with different - * destinations, to check preservation for all registers. - */ -#define BPF_TEST_CLOBBER_ALU(alu, op, dst, src) \ - { \ - #alu "_" #op " to " #dst ": no clobbering", \ - .u.insns_int = { \ - BPF_ALU64_IMM(BPF_MOV, R0, R0), \ - BPF_ALU64_IMM(BPF_MOV, R1, R1), \ - BPF_ALU64_IMM(BPF_MOV, R2, R2), \ - BPF_ALU64_IMM(BPF_MOV, R3, R3), \ - BPF_ALU64_IMM(BPF_MOV, R4, R4), \ - BPF_ALU64_IMM(BPF_MOV, R5, R5), \ - BPF_ALU64_IMM(BPF_MOV, R6, R6), \ - BPF_ALU64_IMM(BPF_MOV, R7, R7), \ - BPF_ALU64_IMM(BPF_MOV, R8, R8), \ - BPF_ALU64_IMM(BPF_MOV, R9, R9), \ - BPF_##alu(BPF_ ##op, dst, src), \ - BPF_ALU32_IMM(BPF_MOV, dst, dst), \ - BPF_JMP_IMM(BPF_JNE, R0, R0, 10), \ - BPF_JMP_IMM(BPF_JNE, R1, R1, 9), \ - BPF_JMP_IMM(BPF_JNE, R2, R2, 8), \ - BPF_JMP_IMM(BPF_JNE, R3, R3, 7), \ - BPF_JMP_IMM(BPF_JNE, R4, R4, 6), \ - BPF_JMP_IMM(BPF_JNE, R5, R5, 5), \ - BPF_JMP_IMM(BPF_JNE, R6, R6, 4), \ - BPF_JMP_IMM(BPF_JNE, R7, R7, 3), \ - BPF_JMP_IMM(BPF_JNE, R8, R8, 2), \ - BPF_JMP_IMM(BPF_JNE, R9, R9, 1), \ - BPF_ALU64_IMM(BPF_MOV, R0, 1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 1 } } \ - } - /* ALU64 operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R9, 123456789), - /* ALU32 immediate operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R9, 123456789), - /* ALU64 register operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R9, R1), - /* ALU32 register operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R9, R1), -#undef BPF_TEST_CLOBBER_ALU -#define BPF_TEST_CLOBBER_ATOMIC(width, op) \ - { \ - "Atomic_" #width " " #op ": no clobbering", \ - .u.insns_int = { \ - BPF_ALU64_IMM(BPF_MOV, R0, 0), \ - BPF_ALU64_IMM(BPF_MOV, R1, 1), \ - BPF_ALU64_IMM(BPF_MOV, R2, 2), \ - BPF_ALU64_IMM(BPF_MOV, R3, 3), \ - BPF_ALU64_IMM(BPF_MOV, R4, 4), \ - BPF_ALU64_IMM(BPF_MOV, R5, 5), \ - BPF_ALU64_IMM(BPF_MOV, R6, 6), \ - BPF_ALU64_IMM(BPF_MOV, R7, 7), \ - BPF_ALU64_IMM(BPF_MOV, R8, 8), \ - BPF_ALU64_IMM(BPF_MOV, R9, 9), \ - BPF_ST_MEM(width, R10, -8, \ - (op) == BPF_CMPXCHG ? 0 : \ - (op) & BPF_FETCH ? 1 : 0), \ - BPF_ATOMIC_OP(width, op, R10, R1, -8), \ - BPF_JMP_IMM(BPF_JNE, R0, 0, 10), \ - BPF_JMP_IMM(BPF_JNE, R1, 1, 9), \ - BPF_JMP_IMM(BPF_JNE, R2, 2, 8), \ - BPF_JMP_IMM(BPF_JNE, R3, 3, 7), \ - BPF_JMP_IMM(BPF_JNE, R4, 4, 6), \ - BPF_JMP_IMM(BPF_JNE, R5, 5, 5), \ - BPF_JMP_IMM(BPF_JNE, R6, 6, 4), \ - BPF_JMP_IMM(BPF_JNE, R7, 7, 3), \ - BPF_JMP_IMM(BPF_JNE, R8, 8, 2), \ - BPF_JMP_IMM(BPF_JNE, R9, 9, 1), \ - BPF_ALU64_IMM(BPF_MOV, R0, 1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 1 } }, \ - .stack_depth = 8, \ - } - /* 64-bit atomic operations, register clobbering */ - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XCHG), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_CMPXCHG), - /* 32-bit atomic operations, register clobbering */ - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XCHG), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_CMPXCHG), -#undef BPF_TEST_CLOBBER_ATOMIC - /* Checking that ALU32 src is not zero extended in place */ -#define BPF_ALU32_SRC_ZEXT(op) \ - { \ - "ALU32_" #op "_X: src preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R1, 0x0123456789acbdefULL),\ - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL),\ - BPF_ALU64_REG(BPF_MOV, R0, R1), \ - BPF_ALU32_REG(BPF_##op, R2, R1), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_ALU32_SRC_ZEXT(MOV), - BPF_ALU32_SRC_ZEXT(AND), - BPF_ALU32_SRC_ZEXT(OR), - BPF_ALU32_SRC_ZEXT(XOR), - BPF_ALU32_SRC_ZEXT(ADD), - BPF_ALU32_SRC_ZEXT(SUB), - BPF_ALU32_SRC_ZEXT(MUL), - BPF_ALU32_SRC_ZEXT(DIV), - BPF_ALU32_SRC_ZEXT(MOD), -#undef BPF_ALU32_SRC_ZEXT - /* Checking that ATOMIC32 src is not zero extended in place */ -#define BPF_ATOMIC32_SRC_ZEXT(op) \ - { \ - "ATOMIC_W_" #op ": src preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ST_MEM(BPF_W, R10, -4, 0), \ - BPF_ATOMIC_OP(BPF_W, BPF_##op, R10, R1, -4), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 8, \ - } - BPF_ATOMIC32_SRC_ZEXT(ADD), - BPF_ATOMIC32_SRC_ZEXT(AND), - BPF_ATOMIC32_SRC_ZEXT(OR), - BPF_ATOMIC32_SRC_ZEXT(XOR), -#undef BPF_ATOMIC32_SRC_ZEXT - /* Checking that CMPXCHG32 src is not zero extended in place */ - { - "ATOMIC_W_CMPXCHG: src preserved in zext", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789acbdefULL), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_REG(BPF_MOV, R0, 0), - BPF_ST_MEM(BPF_W, R10, -4, 0), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R1, -4), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_REG(BPF_OR, R1, R2), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* Checking that JMP32 immediate src is not zero extended in place */ -#define BPF_JMP32_IMM_ZEXT(op) \ - { \ - "JMP32_" #op "_K: operand preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_JMP32_IMM(BPF_##op, R0, 1234, 1), \ - BPF_JMP_A(0), /* Nop */ \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_JMP32_IMM_ZEXT(JEQ), - BPF_JMP32_IMM_ZEXT(JNE), - BPF_JMP32_IMM_ZEXT(JSET), - BPF_JMP32_IMM_ZEXT(JGT), - BPF_JMP32_IMM_ZEXT(JGE), - BPF_JMP32_IMM_ZEXT(JLT), - BPF_JMP32_IMM_ZEXT(JLE), - BPF_JMP32_IMM_ZEXT(JSGT), - BPF_JMP32_IMM_ZEXT(JSGE), - BPF_JMP32_IMM_ZEXT(JSLT), - BPF_JMP32_IMM_ZEXT(JSLE), -#undef BPF_JMP2_IMM_ZEXT - /* Checking that JMP32 dst & src are not zero extended in place */ -#define BPF_JMP32_REG_ZEXT(op) \ - { \ - "JMP32_" #op "_X: operands preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ - BPF_LD_IMM64(R1, 0xfedcba9876543210ULL),\ - BPF_ALU64_REG(BPF_MOV, R2, R0), \ - BPF_ALU64_REG(BPF_MOV, R3, R1), \ - BPF_JMP32_IMM(BPF_##op, R0, R1, 1), \ - BPF_JMP_A(0), /* Nop */ \ - BPF_ALU64_REG(BPF_SUB, R0, R2), \ - BPF_ALU64_REG(BPF_SUB, R1, R3), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_JMP32_REG_ZEXT(JEQ), - BPF_JMP32_REG_ZEXT(JNE), - BPF_JMP32_REG_ZEXT(JSET), - BPF_JMP32_REG_ZEXT(JGT), - BPF_JMP32_REG_ZEXT(JGE), - BPF_JMP32_REG_ZEXT(JLT), - BPF_JMP32_REG_ZEXT(JLE), - BPF_JMP32_REG_ZEXT(JSGT), - BPF_JMP32_REG_ZEXT(JSGE), - BPF_JMP32_REG_ZEXT(JSLT), - BPF_JMP32_REG_ZEXT(JSLE), -#undef BPF_JMP2_REG_ZEXT - /* ALU64 K register combinations */ - { - "ALU64_MOV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_imm_regs, - }, - { - "ALU64_AND_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_imm_regs, - }, - { - "ALU64_OR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_imm_regs, - }, - { - "ALU64_XOR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_imm_regs, - }, - { - "ALU64_LSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_imm_regs, - }, - { - "ALU64_RSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_imm_regs, - }, - { - "ALU64_ARSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_imm_regs, - }, - { - "ALU64_ADD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_imm_regs, - }, - { - "ALU64_SUB_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_imm_regs, - }, - { - "ALU64_MUL_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_imm_regs, - }, - { - "ALU64_DIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_imm_regs, - }, - { - "ALU64_MOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_imm_regs, - }, - { - "ALU64_SDIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sdiv_imm_regs, - }, - { - "ALU64_SMOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_smod_imm_regs, - }, - /* ALU32 K registers */ - { - "ALU32_MOV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_imm_regs, - }, - { - "ALU32_AND_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_imm_regs, - }, - { - "ALU32_OR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_imm_regs, - }, - { - "ALU32_XOR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_imm_regs, - }, - { - "ALU32_LSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_imm_regs, - }, - { - "ALU32_RSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_imm_regs, - }, - { - "ALU32_ARSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_imm_regs, - }, - { - "ALU32_ADD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_imm_regs, - }, - { - "ALU32_SUB_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_imm_regs, - }, - { - "ALU32_MUL_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_imm_regs, - }, - { - "ALU32_DIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_imm_regs, - }, - { - "ALU32_MOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_imm_regs, - }, - { - "ALU32_SDIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sdiv_imm_regs, - }, - { - "ALU32_SMOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_smod_imm_regs, - }, - /* ALU64 X register combinations */ - { - "ALU64_MOV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_reg_pairs, - }, - { - "ALU64_AND_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_reg_pairs, - }, - { - "ALU64_OR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_reg_pairs, - }, - { - "ALU64_XOR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_reg_pairs, - }, - { - "ALU64_LSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_reg_pairs, - }, - { - "ALU64_RSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_reg_pairs, - }, - { - "ALU64_ARSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_reg_pairs, - }, - { - "ALU64_ADD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_reg_pairs, - }, - { - "ALU64_SUB_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_reg_pairs, - }, - { - "ALU64_MUL_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_reg_pairs, - }, - { - "ALU64_DIV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_reg_pairs, - }, - { - "ALU64_MOD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_reg_pairs, - }, - /* ALU32 X register combinations */ - { - "ALU32_MOV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_reg_pairs, - }, - { - "ALU32_AND_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_reg_pairs, - }, - { - "ALU32_OR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_reg_pairs, - }, - { - "ALU32_XOR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_reg_pairs, - }, - { - "ALU32_LSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_reg_pairs, - }, - { - "ALU32_RSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_reg_pairs, - }, - { - "ALU32_ARSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_reg_pairs, - }, - { - "ALU32_ADD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_reg_pairs, - }, - { - "ALU32_SUB_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_reg_pairs, - }, - { - "ALU32_MUL_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_reg_pairs, - }, - { - "ALU32_DIV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_reg_pairs, - }, - { - "ALU32_MOD_X register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_reg_pairs, - }, - /* Exhaustive test of ALU64 shift operations */ - { - "ALU64_LSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_imm, - }, - { - "ALU64_RSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_imm, - }, - { - "ALU64_ARSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_imm, - }, - { - "ALU64_LSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_reg, - }, - { - "ALU64_RSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_reg, - }, - { - "ALU64_ARSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_reg, - }, - /* Exhaustive test of ALU32 shift operations */ - { - "ALU32_LSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_imm, - }, - { - "ALU32_RSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_imm, - }, - { - "ALU32_ARSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_imm, - }, - { - "ALU32_LSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_reg, - }, - { - "ALU32_RSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_reg, - }, - { - "ALU32_ARSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_reg, - }, - /* - * Exhaustive test of ALU64 shift operations when - * source and destination register are the same. - */ - { - "ALU64_LSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_same_reg, - }, - { - "ALU64_RSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_same_reg, - }, - { - "ALU64_ARSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_same_reg, - }, - /* - * Exhaustive test of ALU32 shift operations when - * source and destination register are the same. - */ - { - "ALU32_LSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_same_reg, - }, - { - "ALU32_RSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_same_reg, - }, - { - "ALU32_ARSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_same_reg, - }, - /* ALU64 immediate magnitudes */ - { - "ALU64_MOV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_AND_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_OR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_XOR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_ADD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SUB_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MUL_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_DIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SDIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sdiv_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SMOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_smod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU32 immediate magnitudes */ - { - "ALU32_MOV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_AND_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_OR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_XOR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_ADD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SUB_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MUL_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_DIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SDIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sdiv_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SMOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_smod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU64 register magnitudes */ - { - "ALU64_MOV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_AND_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_OR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_XOR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_ADD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SUB_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MUL_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_DIV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MOD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU32 register magnitudes */ - { - "ALU32_MOV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_AND_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_OR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_XOR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_ADD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SUB_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MUL_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_DIV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MOD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* LD_IMM64 immediate magnitudes and byte patterns */ - { - "LD_IMM64: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_magn, - }, - { - "LD_IMM64: checker byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_checker, - }, - { - "LD_IMM64: random positive and zero byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_pos_zero, - }, - { - "LD_IMM64: random negative and zero byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_neg_zero, - }, - { - "LD_IMM64: random positive and negative byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_pos_neg, - }, - /* 64-bit ATOMIC register combinations */ - { - "ATOMIC_DW_ADD: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_AND: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_OR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XOR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_ADD_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_AND_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_OR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XOR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xchg_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_CMPXCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs, - .stack_depth = 8, - }, - /* 32-bit ATOMIC register combinations */ - { - "ATOMIC_W_ADD: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_AND: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_OR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XOR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_ADD_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_AND_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_OR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XOR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xchg_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_CMPXCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs, - .stack_depth = 8, - }, - /* 64-bit ATOMIC magnitudes */ - { - "ATOMIC_DW_ADD: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_AND: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_OR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XOR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_ADD_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_AND_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_OR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XOR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xchg, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_CMPXCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_cmpxchg64, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* 32-bit atomic magnitudes */ - { - "ATOMIC_W_ADD: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_AND: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_OR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XOR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_ADD_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_AND_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_OR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XOR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xchg, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_CMPXCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_cmpxchg32, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP immediate magnitudes */ - { - "JMP_JSET_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jset_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JEQ_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jeq_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JNE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jne_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jlt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jslt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP register magnitudes */ - { - "JMP_JSET_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jset_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JEQ_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jeq_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JNE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jne_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jlt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jslt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP32 immediate magnitudes */ - { - "JMP32_JSET_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jset_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JEQ_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jeq_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JNE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jne_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jlt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jslt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP32 register magnitudes */ - { - "JMP32_JSET_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jset_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JEQ_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jeq_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JNE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jne_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jlt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jslt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* Conditional jumps with constant decision */ - { - "JMP_JSET_K: imm = 0 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JSET, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLT_K: imm = 0 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JLT, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JGE_K: imm = 0 -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JGE, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: imm = 0xffffffff -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JGT, R1, U32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLE_K: imm = 0xffffffff -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JLE, R1, U32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP32_JSGT_K: imm = 0x7fffffff -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSGT, R1, S32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP32_JSGE_K: imm = -0x80000000 -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSGE, R1, S32_MIN, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP32_JSLT_K: imm = -0x80000000 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSLT, R1, S32_MIN, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP32_JSLE_K: imm = 0x7fffffff -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSLE, R1, S32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JEQ_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JEQ, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JGE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JLE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSGE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSLE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JNE_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JNE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JGT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JGT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JLT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JSGT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSGT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JSLT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSLT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - /* Short relative jumps */ - { - "Short relative jump: offset=0", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=1", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=2", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=3", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 3), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=4", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 4), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - /* Conditional branch conversions */ - { - "Long conditional jump: taken at runtime (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_taken_32, - }, - { - "Long conditional jump: not taken at runtime (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_not_taken_32, - }, - { - "Long conditional jump: always taken, known at JIT time (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_always_taken_32, - }, - { - "Long conditional jump: never taken, known at JIT time (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_never_taken_32, - }, - { - "Long conditional jump: taken at runtime", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_taken, - }, - { - "Long conditional jump: not taken at runtime", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_not_taken, - }, - { - "Long conditional jump: always taken, known at JIT time", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_always_taken, - }, - { - "Long conditional jump: never taken, known at JIT time", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_never_taken, - }, - /* Staggered jump sequences, immediate */ - { - "Staggered jumps: JMP_JA", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_ja, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JEQ_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JNE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSET_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, register */ - { - "Staggered jumps: JMP_JEQ_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JNE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSET_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, JMP32 immediate */ - { - "Staggered jumps: JMP32_JEQ_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JNE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSET_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, JMP32 register */ - { - "Staggered jumps: JMP32_JEQ_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JNE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSET_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, -}; - -static struct net_device dev; - -static struct sk_buff *populate_skb(char *buf, int size) -{ - struct sk_buff *skb; - - if (size >= MAX_DATA) - return NULL; - - skb = alloc_skb(MAX_DATA, GFP_KERNEL); - if (!skb) - return NULL; - - __skb_put_data(skb, buf, size); - - /* Initialize a fake skb with test pattern. */ - skb_reset_mac_header(skb); - skb->protocol = htons(ETH_P_IP); - skb->pkt_type = SKB_TYPE; - skb->mark = SKB_MARK; - skb->hash = SKB_HASH; - skb->queue_mapping = SKB_QUEUE_MAP; - skb->vlan_tci = SKB_VLAN_TCI; - skb->vlan_proto = htons(ETH_P_IP); - dev_net_set(&dev, &init_net); - skb->dev = &dev; - skb->dev->ifindex = SKB_DEV_IFINDEX; - skb->dev->type = SKB_DEV_TYPE; - skb_set_network_header(skb, min(size, ETH_HLEN)); - - return skb; -} - -static void *generate_test_data(struct bpf_test *test, int sub) -{ - struct sk_buff *skb; - struct page *page; - - if (test->aux & FLAG_NO_DATA) - return NULL; - - if (test->aux & FLAG_LARGE_MEM) - return kmalloc(test->test[sub].data_size, GFP_KERNEL); - - /* Test case expects an skb, so populate one. Various - * subtests generate skbs of different sizes based on - * the same data. - */ - skb = populate_skb(test->data, test->test[sub].data_size); - if (!skb) - return NULL; - - if (test->aux & FLAG_SKB_FRAG) { - /* - * when the test requires a fragmented skb, add a - * single fragment to the skb, filled with - * test->frag_data. - */ - page = alloc_page(GFP_KERNEL); - if (!page) - goto err_kfree_skb; - - memcpy(page_address(page), test->frag_data, MAX_DATA); - skb_add_rx_frag(skb, 0, page, 0, MAX_DATA, MAX_DATA); - } - - return skb; -err_kfree_skb: - kfree_skb(skb); - return NULL; -} - -static void release_test_data(const struct bpf_test *test, void *data) -{ - if (test->aux & FLAG_NO_DATA) - return; - - if (test->aux & FLAG_LARGE_MEM) - kfree(data); - else - kfree_skb(data); -} - -static int filter_length(int which) -{ - struct sock_filter *fp; - int len; - - if (tests[which].fill_helper) - return tests[which].u.ptr.len; - - fp = tests[which].u.insns; - for (len = MAX_INSNS - 1; len > 0; --len) - if (fp[len].code != 0 || fp[len].k != 0) - break; - - return len + 1; -} - -static void *filter_pointer(int which) -{ - if (tests[which].fill_helper) - return tests[which].u.ptr.insns; - else - return tests[which].u.insns; -} - -static struct bpf_prog *generate_filter(int which, int *err) -{ - __u8 test_type = tests[which].aux & TEST_TYPE_MASK; - unsigned int flen = filter_length(which); - void *fptr = filter_pointer(which); - struct sock_fprog_kern fprog; - struct bpf_prog *fp; - - switch (test_type) { - case CLASSIC: - fprog.filter = fptr; - fprog.len = flen; - - *err = bpf_prog_create(&fp, &fprog); - if (tests[which].aux & FLAG_EXPECTED_FAIL) { - if (*err == tests[which].expected_errcode) { - pr_cont("PASS\n"); - /* Verifier rejected filter as expected. */ - *err = 0; - return NULL; - } else { - pr_cont("UNEXPECTED_PASS\n"); - /* Verifier didn't reject the test that's - * bad enough, just return! - */ - *err = -EINVAL; - return NULL; - } - } - if (*err) { - pr_cont("FAIL to prog_create err=%d len=%d\n", - *err, fprog.len); - return NULL; - } - break; - - case INTERNAL: - fp = bpf_prog_alloc(bpf_prog_size(flen), 0); - if (fp == NULL) { - pr_cont("UNEXPECTED_FAIL no memory left\n"); - *err = -ENOMEM; - return NULL; - } - - fp->len = flen; - /* Type doesn't really matter here as long as it's not unspec. */ - fp->type = BPF_PROG_TYPE_SOCKET_FILTER; - memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn)); - fp->aux->stack_depth = tests[which].stack_depth; - fp->aux->verifier_zext = !!(tests[which].aux & - FLAG_VERIFIER_ZEXT); - - /* We cannot error here as we don't need type compatibility - * checks. - */ - fp = bpf_prog_select_runtime(fp, err); - if (*err) { - pr_cont("FAIL to select_runtime err=%d\n", *err); - return NULL; - } - break; - } - - *err = 0; - return fp; -} - -static void release_filter(struct bpf_prog *fp, int which) -{ - __u8 test_type = tests[which].aux & TEST_TYPE_MASK; - - switch (test_type) { - case CLASSIC: - bpf_prog_destroy(fp); - break; - case INTERNAL: - bpf_prog_free(fp); - break; - } -} - -static int __run_one(const struct bpf_prog *fp, const void *data, - int runs, u64 *duration) -{ - u64 start, finish; - int ret = 0, i; - - migrate_disable(); - start = ktime_get_ns(); - - for (i = 0; i < runs; i++) - ret = bpf_prog_run(fp, data); - - finish = ktime_get_ns(); - migrate_enable(); - - *duration = finish - start; - do_div(*duration, runs); - - return ret; -} - -static int run_one(const struct bpf_prog *fp, struct bpf_test *test) -{ - int err_cnt = 0, i, runs = MAX_TESTRUNS; - - if (test->nr_testruns) - runs = min(test->nr_testruns, MAX_TESTRUNS); - - for (i = 0; i < MAX_SUBTESTS; i++) { - void *data; - u64 duration; - u32 ret; - - /* - * NOTE: Several sub-tests may be present, in which case - * a zero {data_size, result} tuple indicates the end of - * the sub-test array. The first test is always run, - * even if both data_size and result happen to be zero. - */ - if (i > 0 && - test->test[i].data_size == 0 && - test->test[i].result == 0) - break; - - data = generate_test_data(test, i); - if (!data && !(test->aux & FLAG_NO_DATA)) { - pr_cont("data generation failed "); - err_cnt++; - break; - } - ret = __run_one(fp, data, runs, &duration); - release_test_data(test, data); - - if (ret == test->test[i].result) { - pr_cont("%lld ", duration); - } else { - s32 res = test->test[i].result; - - pr_cont("ret %d != %d (%#x != %#x)", - ret, res, ret, res); - err_cnt++; - } - } - - return err_cnt; -} - -static char test_name[64]; -module_param_string(test_name, test_name, sizeof(test_name), 0); - -static int test_id = -1; -module_param(test_id, int, 0); - -static int test_range[2] = { 0, INT_MAX }; -module_param_array(test_range, int, NULL, 0); - -static bool exclude_test(int test_id) -{ - return test_id < test_range[0] || test_id > test_range[1]; -} - -static __init struct sk_buff *build_test_skb(void) -{ - u32 headroom = NET_SKB_PAD + NET_IP_ALIGN + ETH_HLEN; - struct sk_buff *skb[2]; - struct page *page[2]; - int i, data_size = 8; - - for (i = 0; i < 2; i++) { - page[i] = alloc_page(GFP_KERNEL); - if (!page[i]) { - if (i == 0) - goto err_page0; - else - goto err_page1; - } - - /* this will set skb[i]->head_frag */ - skb[i] = dev_alloc_skb(headroom + data_size); - if (!skb[i]) { - if (i == 0) - goto err_skb0; - else - goto err_skb1; - } - - skb_reserve(skb[i], headroom); - skb_put(skb[i], data_size); - skb[i]->protocol = htons(ETH_P_IP); - skb_reset_network_header(skb[i]); - skb_set_mac_header(skb[i], -ETH_HLEN); - - skb_add_rx_frag(skb[i], 0, page[i], 0, 64, 64); - // skb_headlen(skb[i]): 8, skb[i]->head_frag = 1 - } - - /* setup shinfo */ - skb_shinfo(skb[0])->gso_size = 1448; - skb_shinfo(skb[0])->gso_type = SKB_GSO_TCPV4; - skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY; - skb_shinfo(skb[0])->gso_segs = 0; - skb_shinfo(skb[0])->frag_list = skb[1]; - skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000; - - /* adjust skb[0]'s len */ - skb[0]->len += skb[1]->len; - skb[0]->data_len += skb[1]->data_len; - skb[0]->truesize += skb[1]->truesize; - - return skb[0]; - -err_skb1: - __free_page(page[1]); -err_page1: - kfree_skb(skb[0]); -err_skb0: - __free_page(page[0]); -err_page0: - return NULL; -} - -static __init struct sk_buff *build_test_skb_linear_no_head_frag(void) -{ - unsigned int alloc_size = 2000; - unsigned int headroom = 102, doffset = 72, data_size = 1308; - struct sk_buff *skb[2]; - int i; - - /* skbs linked in a frag_list, both with linear data, with head_frag=0 - * (data allocated by kmalloc), both have tcp data of 1308 bytes - * (total payload is 2616 bytes). - * Data offset is 72 bytes (40 ipv6 hdr, 32 tcp hdr). Some headroom. - */ - for (i = 0; i < 2; i++) { - skb[i] = alloc_skb(alloc_size, GFP_KERNEL); - if (!skb[i]) { - if (i == 0) - goto err_skb0; - else - goto err_skb1; - } - - skb[i]->protocol = htons(ETH_P_IPV6); - skb_reserve(skb[i], headroom); - skb_put(skb[i], doffset + data_size); - skb_reset_network_header(skb[i]); - if (i == 0) - skb_reset_mac_header(skb[i]); - else - skb_set_mac_header(skb[i], -ETH_HLEN); - __skb_pull(skb[i], doffset); - } - - /* setup shinfo. - * mimic bpf_skb_proto_4_to_6, which resets gso_segs and assigns a - * reduced gso_size. - */ - skb_shinfo(skb[0])->gso_size = 1288; - skb_shinfo(skb[0])->gso_type = SKB_GSO_TCPV6 | SKB_GSO_DODGY; - skb_shinfo(skb[0])->gso_segs = 0; - skb_shinfo(skb[0])->frag_list = skb[1]; - - /* adjust skb[0]'s len */ - skb[0]->len += skb[1]->len; - skb[0]->data_len += skb[1]->len; - skb[0]->truesize += skb[1]->truesize; - - return skb[0]; - -err_skb1: - kfree_skb(skb[0]); -err_skb0: - return NULL; -} - -struct skb_segment_test { - const char *descr; - struct sk_buff *(*build_skb)(void); - netdev_features_t features; -}; - -static struct skb_segment_test skb_segment_tests[] __initconst = { - { - .descr = "gso_with_rx_frags", - .build_skb = build_test_skb, - .features = NETIF_F_SG | NETIF_F_GSO_PARTIAL | NETIF_F_IP_CSUM | - NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM - }, - { - .descr = "gso_linear_no_head_frag", - .build_skb = build_test_skb_linear_no_head_frag, - .features = NETIF_F_SG | NETIF_F_FRAGLIST | - NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_GSO | - NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | - NETIF_F_HW_VLAN_STAG_TX - } -}; - -static __init int test_skb_segment_single(const struct skb_segment_test *test) -{ - struct sk_buff *skb, *segs; - int ret = -1; - - skb = test->build_skb(); - if (!skb) { - pr_info("%s: failed to build_test_skb", __func__); - goto done; - } - - segs = skb_segment(skb, test->features); - if (!IS_ERR(segs)) { - kfree_skb_list(segs); - ret = 0; - } - kfree_skb(skb); -done: - return ret; -} - -static __init int test_skb_segment(void) -{ - int i, err_cnt = 0, pass_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { - const struct skb_segment_test *test = &skb_segment_tests[i]; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, test->descr); - - if (test_skb_segment_single(test)) { - pr_cont("FAIL\n"); - err_cnt++; - } else { - pr_cont("PASS\n"); - pass_cnt++; - } - } - - pr_info("%s: Summary: %d PASSED, %d FAILED\n", __func__, - pass_cnt, err_cnt); - return err_cnt ? -EINVAL : 0; -} - -static __init int test_bpf(void) -{ - int i, err_cnt = 0, pass_cnt = 0; - int jit_cnt = 0, run_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - struct bpf_prog *fp; - int err; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, tests[i].descr); - - if (tests[i].fill_helper && - tests[i].fill_helper(&tests[i]) < 0) { - pr_cont("FAIL to prog_fill\n"); - continue; - } - - fp = generate_filter(i, &err); - - if (tests[i].fill_helper) { - kfree(tests[i].u.ptr.insns); - tests[i].u.ptr.insns = NULL; - } - - if (fp == NULL) { - if (err == 0) { - pass_cnt++; - continue; - } - err_cnt++; - continue; - } - - pr_cont("jited:%u ", fp->jited); - - run_cnt++; - if (fp->jited) - jit_cnt++; - - err = run_one(fp, &tests[i]); - release_filter(fp, i); - - if (err) { - pr_cont("FAIL (%d times)\n", err); - err_cnt++; - } else { - pr_cont("PASS\n"); - pass_cnt++; - } - } - - pr_info("Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", - pass_cnt, err_cnt, jit_cnt, run_cnt); - - return err_cnt ? -EINVAL : 0; -} - -struct tail_call_test { - const char *descr; - struct bpf_insn insns[MAX_INSNS]; - int flags; - int result; - int stack_depth; - bool has_tail_call; -}; - -/* Flags that can be passed to tail call test cases */ -#define FLAG_NEED_STATE BIT(0) -#define FLAG_RESULT_IN_STATE BIT(1) - -/* - * Magic marker used in test snippets for tail calls below. - * BPF_LD/MOV to R2 and R2 with this immediate value is replaced - * with the proper values by the test runner. - */ -#define TAIL_CALL_MARKER 0x7a11ca11 - -/* Special offset to indicate a NULL call target */ -#define TAIL_CALL_NULL 0x7fff - -/* Special offset to indicate an out-of-range index */ -#define TAIL_CALL_INVALID 0x7ffe - -#define TAIL_CALL(offset) \ - BPF_LD_IMM64(R2, TAIL_CALL_MARKER), \ - BPF_RAW_INSN(BPF_ALU | BPF_MOV | BPF_K, R3, 0, \ - offset, TAIL_CALL_MARKER), \ - BPF_JMP_IMM(BPF_TAIL_CALL, 0, 0, 0) - -/* - * A test function to be called from a BPF program, clobbering a lot of - * CPU registers in the process. A JITed BPF program calling this function - * must save and restore any caller-saved registers it uses for internal - * state, for example the current tail call count. - */ -BPF_CALL_1(bpf_test_func, u64, arg) -{ - char buf[64]; - long a = 0; - long b = 1; - long c = 2; - long d = 3; - long e = 4; - long f = 5; - long g = 6; - long h = 7; - - return snprintf(buf, sizeof(buf), - "%ld %lu %lx %ld %lu %lx %ld %lu %x", - a, b, c, d, e, f, g, h, (int)arg); -} -#define BPF_FUNC_test_func __BPF_FUNC_MAX_ID - -/* - * Tail call tests. Each test case may call any other test in the table, - * including itself, specified as a relative index offset from the calling - * test. The index TAIL_CALL_NULL can be used to specify a NULL target - * function to test the JIT error path. Similarly, the index TAIL_CALL_INVALID - * results in a target index that is out of range. - */ -static struct tail_call_test tail_call_tests[] = { - { - "Tail call leaf", - .insns = { - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - }, - .result = 1, - }, - { - "Tail call 2", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 2), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 3, - .has_tail_call = true, - }, - { - "Tail call 3", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 3), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 6, - .has_tail_call = true, - }, - { - "Tail call 4", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 4), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 10, - .has_tail_call = true, - }, - { - "Tail call load/store leaf", - .insns = { - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_REG(BPF_MOV, R3, BPF_REG_FP), - BPF_STX_MEM(BPF_DW, R3, R1, -8), - BPF_STX_MEM(BPF_DW, R3, R2, -16), - BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 3), - BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -16), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - .result = 0, - .stack_depth = 32, - }, - { - "Tail call load/store", - .insns = { - BPF_ALU64_IMM(BPF_MOV, R0, 3), - BPF_STX_MEM(BPF_DW, BPF_REG_FP, R0, -8), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 0, - .stack_depth = 16, - .has_tail_call = true, - }, - { - "Tail call error path, max count reached", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(0), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call count preserved across function calls", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_CALL_REL(BPF_FUNC_get_numa_node_id), - BPF_CALL_REL(BPF_FUNC_ktime_get_ns), - BPF_CALL_REL(BPF_FUNC_ktime_get_boot_ns), - BPF_CALL_REL(BPF_FUNC_ktime_get_coarse_ns), - BPF_CALL_REL(BPF_FUNC_jiffies64), - BPF_CALL_REL(BPF_FUNC_test_func), - BPF_LDX_MEM(BPF_DW, R1, R10, -8), - BPF_ALU32_REG(BPF_MOV, R0, R1), - TAIL_CALL(0), - BPF_EXIT_INSN(), - }, - .stack_depth = 8, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call error path, NULL target", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(TAIL_CALL_NULL), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call error path, index out of range", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(TAIL_CALL_INVALID), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = MAX_TESTRUNS, - .has_tail_call = true, - }, -}; - -static void __init destroy_tail_call_tests(struct bpf_array *progs) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) - if (progs->ptrs[i]) - bpf_prog_free(progs->ptrs[i]); - kfree(progs); -} - -static __init int prepare_tail_call_tests(struct bpf_array **pprogs) -{ - int ntests = ARRAY_SIZE(tail_call_tests); - struct bpf_array *progs; - int which, err; - - /* Allocate the table of programs to be used for tail calls */ - progs = kzalloc_flex(*progs, ptrs, ntests + 1); - if (!progs) - goto out_nomem; - - /* Create all eBPF programs and populate the table */ - for (which = 0; which < ntests; which++) { - struct tail_call_test *test = &tail_call_tests[which]; - struct bpf_prog *fp; - int len, i; - - /* Compute the number of program instructions */ - for (len = 0; len < MAX_INSNS; len++) { - struct bpf_insn *insn = &test->insns[len]; - - if (len < MAX_INSNS - 1 && - insn->code == (BPF_LD | BPF_DW | BPF_IMM)) - len++; - if (insn->code == 0) - break; - } - - /* Allocate and initialize the program */ - fp = bpf_prog_alloc(bpf_prog_size(len), 0); - if (!fp) - goto out_nomem; - - fp->len = len; - fp->type = BPF_PROG_TYPE_SOCKET_FILTER; - fp->aux->stack_depth = test->stack_depth; - fp->aux->tail_call_reachable = test->has_tail_call; - memcpy(fp->insnsi, test->insns, len * sizeof(struct bpf_insn)); - - /* Relocate runtime tail call offsets and addresses */ - for (i = 0; i < len; i++) { - struct bpf_insn *insn = &fp->insnsi[i]; - long addr = 0; - - switch (insn->code) { - case BPF_LD | BPF_DW | BPF_IMM: - if (insn->imm != TAIL_CALL_MARKER) - break; - insn[0].imm = (u32)(long)progs; - insn[1].imm = ((u64)(long)progs) >> 32; - break; - - case BPF_ALU | BPF_MOV | BPF_K: - if (insn->imm != TAIL_CALL_MARKER) - break; - if (insn->off == TAIL_CALL_NULL) - insn->imm = ntests; - else if (insn->off == TAIL_CALL_INVALID) - insn->imm = ntests + 1; - else - insn->imm = which + insn->off; - insn->off = 0; - break; - - case BPF_JMP | BPF_CALL: - if (insn->src_reg != BPF_PSEUDO_CALL) - break; - switch (insn->imm) { - case BPF_FUNC_get_numa_node_id: - addr = (long)&numa_node_id; - break; - case BPF_FUNC_ktime_get_ns: - addr = (long)&ktime_get_ns; - break; - case BPF_FUNC_ktime_get_boot_ns: - addr = (long)&ktime_get_boot_fast_ns; - break; - case BPF_FUNC_ktime_get_coarse_ns: - addr = (long)&ktime_get_coarse_ns; - break; - case BPF_FUNC_jiffies64: - addr = (long)&get_jiffies_64; - break; - case BPF_FUNC_test_func: - addr = (long)&bpf_test_func; - break; - default: - err = -EFAULT; - goto out_err; - } - *insn = BPF_EMIT_CALL(addr); - if ((long)__bpf_call_base + insn->imm != addr) - *insn = BPF_JMP_A(0); /* Skip: NOP */ - break; - } - } - - fp = bpf_prog_select_runtime(fp, &err); - if (err) - goto out_err; - - progs->ptrs[which] = fp; - } - - /* The last entry contains a NULL program pointer */ - progs->map.max_entries = ntests + 1; - *pprogs = progs; - return 0; - -out_nomem: - err = -ENOMEM; - -out_err: - if (progs) - destroy_tail_call_tests(progs); - return err; -} - -static __init int test_tail_calls(struct bpf_array *progs) -{ - int i, err_cnt = 0, pass_cnt = 0; - int jit_cnt = 0, run_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { - struct tail_call_test *test = &tail_call_tests[i]; - struct bpf_prog *fp = progs->ptrs[i]; - int *data = NULL; - int state = 0; - u64 duration; - int ret; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, test->descr); - if (!fp) { - err_cnt++; - continue; - } - pr_cont("jited:%u ", fp->jited); - - run_cnt++; - if (fp->jited) - jit_cnt++; - - if (test->flags & FLAG_NEED_STATE) - data = &state; - ret = __run_one(fp, data, MAX_TESTRUNS, &duration); - if (test->flags & FLAG_RESULT_IN_STATE) - ret = state; - if (ret == test->result) { - pr_cont("%lld PASS", duration); - pass_cnt++; - } else { - pr_cont("ret %d != %d FAIL", ret, test->result); - err_cnt++; - } - } - - pr_info("%s: Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", - __func__, pass_cnt, err_cnt, jit_cnt, run_cnt); - - return err_cnt ? -EINVAL : 0; -} - -static char test_suite[32]; -module_param_string(test_suite, test_suite, sizeof(test_suite), 0); - -static __init int find_test_index(const char *test_name) -{ - int i; - - if (!strcmp(test_suite, "test_bpf")) { - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!strcmp(tests[i].descr, test_name)) - return i; - } - } - - if (!strcmp(test_suite, "test_tail_calls")) { - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { - if (!strcmp(tail_call_tests[i].descr, test_name)) - return i; - } - } - - if (!strcmp(test_suite, "test_skb_segment")) { - for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { - if (!strcmp(skb_segment_tests[i].descr, test_name)) - return i; - } - } - - return -1; -} - -static __init int prepare_test_range(void) -{ - int valid_range; - - if (!strcmp(test_suite, "test_bpf")) - valid_range = ARRAY_SIZE(tests); - else if (!strcmp(test_suite, "test_tail_calls")) - valid_range = ARRAY_SIZE(tail_call_tests); - else if (!strcmp(test_suite, "test_skb_segment")) - valid_range = ARRAY_SIZE(skb_segment_tests); - else - return 0; - - if (test_id >= 0) { - /* - * if a test_id was specified, use test_range to - * cover only that test. - */ - if (test_id >= valid_range) { - pr_err("test_bpf: invalid test_id specified for '%s' suite.\n", - test_suite); - return -EINVAL; - } - - test_range[0] = test_id; - test_range[1] = test_id; - } else if (*test_name) { - /* - * if a test_name was specified, find it and setup - * test_range to cover only that test. - */ - int idx = find_test_index(test_name); - - if (idx < 0) { - pr_err("test_bpf: no test named '%s' found for '%s' suite.\n", - test_name, test_suite); - return -EINVAL; - } - test_range[0] = idx; - test_range[1] = idx; - } else if (test_range[0] != 0 || test_range[1] != INT_MAX) { - /* - * check that the supplied test_range is valid. - */ - if (test_range[0] < 0 || test_range[1] >= valid_range) { - pr_err("test_bpf: test_range is out of bound for '%s' suite.\n", - test_suite); - return -EINVAL; - } - - if (test_range[1] < test_range[0]) { - pr_err("test_bpf: test_range is ending before it starts.\n"); - return -EINVAL; - } - } - - return 0; -} - -static int __init test_bpf_init(void) -{ - struct bpf_array *progs = NULL; - int ret; - - if (strlen(test_suite) && - strcmp(test_suite, "test_bpf") && - strcmp(test_suite, "test_tail_calls") && - strcmp(test_suite, "test_skb_segment")) { - pr_err("test_bpf: invalid test_suite '%s' specified.\n", test_suite); - return -EINVAL; - } - - /* - * if test_suite is not specified, but test_id, test_name or test_range - * is specified, set 'test_bpf' as the default test suite. - */ - if (!strlen(test_suite) && - (test_id != -1 || strlen(test_name) || - (test_range[0] != 0 || test_range[1] != INT_MAX))) { - pr_info("test_bpf: set 'test_bpf' as the default test_suite.\n"); - strscpy(test_suite, "test_bpf", sizeof(test_suite)); - } - - ret = prepare_test_range(); - if (ret < 0) - return ret; - - if (!strlen(test_suite) || !strcmp(test_suite, "test_bpf")) { - ret = test_bpf(); - if (ret) - return ret; - } - - if (!strlen(test_suite) || !strcmp(test_suite, "test_tail_calls")) { - ret = prepare_tail_call_tests(&progs); - if (ret) - return ret; - ret = test_tail_calls(progs); - destroy_tail_call_tests(progs); - if (ret) - return ret; - } - - if (!strlen(test_suite) || !strcmp(test_suite, "test_skb_segment")) - return test_skb_segment(); - - return 0; -} - -static void __exit test_bpf_exit(void) -{ -} - -module_init(test_bpf_init); -module_exit(test_bpf_exit); - -MODULE_DESCRIPTION("Testsuite for BPF interpreter and BPF JIT compiler"); -MODULE_LICENSE("GPL");
diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index 716df49..d8f579c 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c
@@ -8,6 +8,8 @@ #include <linux/memcontrol.h> #include <linux/bpf.h> +#include "internal.h" + __bpf_kfunc_start_defs(); /** @@ -159,6 +161,51 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg) mem_cgroup_flush_stats(memcg); } +/** + * bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup + * @memcg: the target memory cgroup to reclaim from. + * @size: the amount of memory to reclaim, in bytes, clamped to + * MEMCG_CHARGE_BATCH. + * @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS, + * MAX_SWAPPINESS], or one of the special modes: -1 to use + * the memcg's own swappiness, or SWAPPINESS_ANON_ONLY to + * reclaim only anon folios. + * + * Performs one proactive reclaim pass on @memcg, like a write to + * memory.reclaim but without retrying until @size is reached. Call it + * repeatedly to reclaim more than one batch. + * + * Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs + * may run with filesystem locks held, which the reclaim path can deadlock + * on via filesystem shrinkers. + * + * Return: The amount of memory reclaimed, in bytes, or a negative error. + */ +__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg, + unsigned long size, + int swappiness) +{ + unsigned long nr_reclaimed; + unsigned long nr_pages; + + if (swappiness != -1 && swappiness != SWAPPINESS_ANON_ONLY) { + if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS) + return -EINVAL; + } + + if (size < PAGE_SIZE) + return -EINVAL; + + nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH); + + nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL, + MEMCG_RECLAIM_MAY_SWAP | + MEMCG_RECLAIM_PROACTIVE, + swappiness == -1 ? NULL : &swappiness); + + return nr_reclaimed * PAGE_SIZE; +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_memcontrol_kfuncs) @@ -174,19 +221,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE) BTF_KFUNCS_END(bpf_memcontrol_kfuncs) +BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs) +BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE) +BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs) + static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = { .owner = THIS_MODULE, .set = &bpf_memcontrol_kfuncs, }; +static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_memcontrol_reclaim_kfuncs, +}; + static int __init bpf_memcontrol_init(void) { int err; err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_memcontrol_kfunc_set); - if (err) + if (err) { pr_warn("error while registering bpf memcontrol kfuncs: %d", err); + return err; + } + + err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &bpf_memcontrol_reclaim_kfunc_set); + if (err) + pr_warn("error registering bpf reclaim kfuncs: %d\n", err); return err; }
diff --git a/mm/internal.h b/mm/internal.h index 05179c4..8c6c9db 100644 --- a/mm/internal.h +++ b/mm/internal.h
@@ -24,6 +24,9 @@ struct folio_batch; +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); + /* mm/workingset.c */ bool workingset_test_recent(void *shadow, bool file, bool *workingset, bool flush); @@ -67,11 +70,13 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) #define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 +enum { + MIN_SWAPPINESS = 0, + MAX_SWAPPINESS = 200, -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + /* Just reclaim from anon folios in proactive memory reclaim */ + SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1, +}; unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages,
diff --git a/mm/memory.c b/mm/memory.c index 7d821b1..797d395 100644 --- a/mm/memory.c +++ b/mm/memory.c
@@ -7302,13 +7302,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, } EXPORT_SYMBOL_GPL(access_process_vm); -#ifdef CONFIG_BPF_SYSCALL /* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, - void *buf, int len, unsigned int gup_flags) +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) { void *old_buf = buf; int err = 0; @@ -7384,44 +7383,6 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, return buf - old_buf; } -/** - * copy_remote_vm_str - copy a string from another process's address space. - * @tsk: the task of the target address space - * @addr: start address to read from - * @buf: destination buffer - * @len: number of bytes to copy - * @gup_flags: flags modifying lookup behaviour - * - * The caller must hold a reference on @mm. - * - * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. - */ -int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags) -{ - struct mm_struct *mm; - int ret; - - if (unlikely(len == 0)) - return 0; - - mm = get_task_mm(tsk); - if (!mm) { - *(char *)buf = '\0'; - return -EFAULT; - } - - ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags); - - mmput(mm); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_remote_vm_str); -#endif /* CONFIG_BPF_SYSCALL */ - /* * Print the name of a VMA. */
diff --git a/mm/nommu.c b/mm/nommu.c index ed44510..91ab65a 100644 --- a/mm/nommu.c +++ b/mm/nommu.c
@@ -1741,13 +1741,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in } EXPORT_SYMBOL_GPL(access_process_vm); -#ifdef CONFIG_BPF_SYSCALL /* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, - void *buf, int len) +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) { unsigned long addr_end; struct vm_area_struct *vma; @@ -1783,44 +1782,6 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, } /** - * copy_remote_vm_str - copy a string from another process's address space. - * @tsk: the task of the target address space - * @addr: start address to read from - * @buf: destination buffer - * @len: number of bytes to copy - * @gup_flags: flags modifying lookup behaviour (unused) - * - * The caller must hold a reference on @mm. - * - * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. - */ -int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags) -{ - struct mm_struct *mm; - int ret; - - if (unlikely(len == 0)) - return 0; - - mm = get_task_mm(tsk); - if (!mm) { - *(char *)buf = '\0'; - return -EFAULT; - } - - ret = __copy_remote_vm_str(mm, addr, buf, len); - - mmput(mm); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_remote_vm_str); -#endif /* CONFIG_BPF_SYSCALL */ - -/** * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode * @inode: The inode to check * @size: The current filesize of the inode
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ff7b9ac..3322f9e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c
@@ -5648,7 +5648,7 @@ EXPORT_SYMBOL(alloc_pages_exact_noprof); * * Return: pointer to the allocated area or %NULL in case of error. */ -void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) +void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) { unsigned int order = get_order(size); struct page *p;
diff --git a/mm/util.c b/mm/util.c index bf0513d..75c0fc2 100644 --- a/mm/util.c +++ b/mm/util.c
@@ -1061,6 +1061,66 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen) return res; } +/** + * copy_remote_mm_str - copy a string from a remote address space. + * @mm: the remote address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * The caller must hold a reference on @mm. + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + if (unlikely(len == 0)) + return 0; + + return __copy_remote_mm_str(mm, addr, buf, len, gup_flags); +} + +/** + * copy_remote_vm_str - copy a string from another process's address space. + * @tsk: the task of the target address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + struct mm_struct *mm; + int ret; + + if (unlikely(len == 0)) + return 0; + + mm = get_task_mm(tsk); + if (!mm) { + *(char *)buf = '\0'; + return -EFAULT; + } + + ret = __copy_remote_mm_str(mm, addr, buf, len, gup_flags); + + mmput(mm); + + return ret; +} +EXPORT_SYMBOL_GPL(copy_remote_vm_str); + int __weak memcmp_pages(struct page *page1, struct page *page2) { char *addr1, *addr2;
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 513354e..3043a73 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c
@@ -1237,7 +1237,13 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, memset(__skb_push(skb, hh_len), 0, hh_len); } - if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) { + /* + * A helper such as bpf_skb_change_tail() may have downgraded the skb + * from CHECKSUM_COMPLETE. skb->csum is then unused by the stack and + * there is nothing to validate. + */ + if ((kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) && + skb->ip_summed == CHECKSUM_COMPLETE) { const int off = skb_network_offset(skb); int len = skb->len - off; __wsum csum; @@ -1650,21 +1656,20 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat ctx.selected_sk = NULL; retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); + + /* Nothing but RCU keeps the selected socket */ + user_ctx->cookie = 0; + if (!ret && ctx.selected_sk) { + if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) + ret = -EOPNOTSUPP; + else + user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); + } bpf_test_timer_leave(&t); if (ret < 0) goto out; - user_ctx->cookie = 0; - if (ctx.selected_sk) { - if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { - ret = -EOPNOTSUPP; - goto out; - } - - user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); - } - ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration); if (!ret) ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
diff --git a/net/core/bpf_ksock.c b/net/core/bpf_ksock.c index e9943ae..37168f4 100644 --- a/net/core/bpf_ksock.c +++ b/net/core/bpf_ksock.c
@@ -37,13 +37,47 @@ static void ksock_release_work_fn(struct work_struct *work) kfree(ks); } -static bool bpf_ksock_has_user_task_context(void) +static int bpf_ksock_addr_to_sockaddr(const union bpf_ksock_addr *addr, + u32 addr_sz, struct sockaddr_storage *sa) { - /* - * Task work can run from do_exit() after exit_nsproxy_namespaces() - * cleared current->nsproxy, while current is still not a kthread. - */ - return !(current->flags & PF_KTHREAD) && current->nsproxy; + if (!addr || addr_sz != sizeof(*addr)) + return -EINVAL; + + /* Kfunc memory arguments may be unaligned. */ + memcpy(sa, addr, sizeof(*addr)); + + switch (sa->ss_family) { + case AF_INET: + return sizeof(struct sockaddr_in); + case AF_INET6: + return sizeof(struct sockaddr_in6); + default: + return -EAFNOSUPPORT; + } +} + +static int ksock_sendto(struct bpf_ksock *ks, const void *data, u32 data_sz, + const union bpf_ksock_addr *addr, u32 addr_sz) +{ + struct sockaddr_storage sa; + struct msghdr msg = { + .msg_flags = MSG_DONTWAIT, + }; + struct kvec iov = { + .iov_base = (void *)data, + .iov_len = data_sz, + }; + int addrlen; + + if (addr) { + addrlen = bpf_ksock_addr_to_sockaddr(addr, addr_sz, &sa); + if (addrlen < 0) + return addrlen; + msg.msg_name = &sa; + msg.msg_namelen = addrlen; + } + + return kernel_sendmsg(ks->sock, &msg, &iov, 1, data_sz); } __bpf_kfunc_start_defs(); @@ -59,7 +93,7 @@ __bpf_kfunc_start_defs(); * This function may sleep (sock_create), so it can only be used * in sleepable BPF programs (SYSCALL). * It cannot be called from a BPF workqueue callback because that callback - * does not retain the invoking task's namespace or security context. + * does not retain the invoking task's namespace. * * @opts: Pointer to struct bpf_ksock_create_opts with socket parameters. * @opts__sz: Size of the opts struct. @@ -74,11 +108,10 @@ bpf_ksock_create(const struct bpf_ksock_create_opts *opts, u32 opts__sz, int err; /* - * sock_create() derives the network namespace, credentials, and cgroup - * from current. Kernel threads, including BPF workqueue callbacks, do - * not carry the context of the task that invoked the BPF program. + * sock_create() derives the network namespace from current and task + * work can run from do_exit() after exit_nsproxy_namespaces() */ - if (!bpf_ksock_has_user_task_context()) { + if ((current->flags & PF_KTHREAD) || !current->nsproxy) { err = -EOPNOTSUPP; goto err_out; } @@ -165,25 +198,9 @@ __bpf_kfunc int bpf_ksock_connect(struct bpf_ksock *ks, struct sockaddr_storage sa; int addrlen; - if (!bpf_ksock_has_user_task_context()) - return -EOPNOTSUPP; - - if (!addr || addr__sz != sizeof(*addr)) - return -EINVAL; - - /* Kfunc memory arguments may be unaligned. */ - memcpy(&sa, addr, sizeof(*addr)); - - switch (sa.ss_family) { - case AF_INET: - addrlen = sizeof(struct sockaddr_in); - break; - case AF_INET6: - addrlen = sizeof(struct sockaddr_in6); - break; - default: - return -EAFNOSUPPORT; - } + addrlen = bpf_ksock_addr_to_sockaddr(addr, addr__sz, &sa); + if (addrlen < 0) + return addrlen; return connect_socket(ks->sock, &sa, addrlen, 0); } @@ -238,21 +255,29 @@ CFI_NOSEAL(bpf_ksock_release_dtor); __bpf_kfunc int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) { - struct msghdr msg = { - .msg_flags = MSG_DONTWAIT, - }; - struct kvec iov = { - .iov_base = (void *)data, - .iov_len = data__sz, - }; - int ret; + return ksock_sendto(ks, data, data__sz, NULL, 0); +} - if (!bpf_ksock_has_user_task_context()) - return -EOPNOTSUPP; - - ret = kernel_sendmsg(ks->sock, &msg, &iov, 1, data__sz); - - return ret; +/** + * bpf_ksock_sendto() - Send data through a BPF kernel socket to a remote address. + * @ks: The BPF kernel socket context. Must be an acquired reference. + * @data: Pointer to the data to send. + * @data__sz: Size of the data to send. + * @addr__nullable: Pointer to an IPv4 or IPv6 socket address, or NULL to use + * the connected destination. + * @addr__sz: Size of the address union, ignored when the address is NULL. + * + * Sends data to the specified address, or to the connected destination when + * the address is NULL, best-effort and nonblocking. This may sleep + * (kernel_sendmsg), so it can only be called from sleepable BPF programs. + * + * Return: Number of bytes sent on success, negative errno on error. + */ +__bpf_kfunc int bpf_ksock_sendto(struct bpf_ksock *ks, const void *data, u32 data__sz, + const union bpf_ksock_addr *addr__nullable, + u32 addr__sz) +{ + return ksock_sendto(ks, data, data__sz, addr__nullable, addr__sz); } __bpf_kfunc_end_defs(); @@ -271,10 +296,16 @@ BTF_KFUNCS_START(ksock_kfunc_btf_ids) BTF_ID_FLAGS(func, bpf_ksock_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_ksock_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_ksock_send, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_ksock_sendto, KF_SLEEPABLE) BTF_KFUNCS_END(ksock_kfunc_btf_ids) #ifdef CONFIG_BPF_LSM -BTF_ID_LIST_SINGLE(bpf_lsm_socket_sendmsg_id, func, bpf_lsm_socket_sendmsg) +BTF_ID_LIST(bpf_lsm_socket_sendmsg_id) +#ifdef CONFIG_SECURITY_NETWORK +BTF_ID(func, bpf_lsm_socket_sendmsg) +#else +BTF_ID_UNUSED +#endif #endif static int bpf_ksock_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id)
diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 1d295a8..1c59300 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c
@@ -164,6 +164,13 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) if (!sk_storage || hlist_empty(&sk_storage->list)) goto out; + /* + * The maps lead to newsk as soon as an element is linked, but + * sk_clone() sets sk_refcnt only later. Until then it holds whatever + * the slab object held before: no reference can be had yet. + */ + refcount_set(&newsk->sk_refcnt, 0); + hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { struct bpf_local_storage_elem *copy_selem; struct bpf_local_storage_map *smap; @@ -692,8 +699,37 @@ struct bpf_iter_seq_sk_storage_map_info { struct bpf_map *map; unsigned int bucket_id; unsigned skip_elems; + struct rcu_head *dead_sks; }; +/* + * sock_put() in two halves. The end of a socket may want process context: + * rfcomm_sock_destruct() takes a mutex. If the iterator drops the last + * reference, it keeps the socket until it has left the RCU read side critical + * section. Nobody uses sk_rcu before sk_destruct(). + */ +static void +bpf_sk_storage_map_seq_put_sk(struct bpf_iter_seq_sk_storage_map_info *info, + struct sock *sk) +{ + if (refcount_dec_and_test(&sk->sk_refcnt)) { + sk->sk_rcu.next = info->dead_sks; + info->dead_sks = &sk->sk_rcu; + } +} + +static void +bpf_sk_storage_map_seq_free_sks(struct bpf_iter_seq_sk_storage_map_info *info) +{ + struct sock *sk; + + while (info->dead_sks) { + sk = container_of(info->dead_sks, struct sock, sk_rcu); + info->dead_sks = sk->sk_rcu.next; + sk_free(sk); + } +} + static struct bpf_local_storage_elem * bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, struct bpf_local_storage_elem *prev_selem) @@ -722,6 +758,7 @@ bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, /* not found, unlock and go to the next bucket */ b = &smap->buckets[bucket_id++]; rcu_read_unlock(); + bpf_sk_storage_map_seq_free_sks(info); skip_elems = 0; break; } @@ -789,12 +826,44 @@ DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, struct bpf_map *map, struct sock *sk, void *value) +/* + * Take a reference on the socket which owns @selem. RCU keeps the element, not + * its owner. As long as the element is linked under the lock of the storage, + * the owner has not got past bpf_sk_storage_free(), unless that has given up + * on the lock. Then it still waits for owner_refcnt. + */ +static struct sock * +bpf_sk_storage_selem_get_sk(struct bpf_local_storage_elem *selem) +{ + struct bpf_local_storage *sk_storage; + struct sock *sk = NULL; + unsigned long flags; + + sk_storage = rcu_dereference(selem->local_storage); + /* reset since ->next() looked: the socket goes away */ + if (!sk_storage) + return NULL; + + if (raw_res_spin_lock_irqsave(&sk_storage->lock, flags)) + return NULL; + + if (!hlist_unhashed(&selem->snode) && + refcount_inc_not_zero(&sk_storage->owner_refcnt)) { + sk = sk_storage->owner; + if (!refcount_inc_not_zero(&sk->sk_refcnt)) + sk = NULL; + bpf_local_storage_unpin_owner(sk_storage); + } + raw_res_spin_unlock_irqrestore(&sk_storage->lock, flags); + + return sk; +} + static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, struct bpf_local_storage_elem *selem) { struct bpf_iter_seq_sk_storage_map_info *info = seq->private; struct bpf_iter__bpf_sk_storage_map ctx = {}; - struct bpf_local_storage *sk_storage; struct bpf_iter_meta meta; struct bpf_prog *prog; int ret = 0; @@ -805,11 +874,14 @@ static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, ctx.meta = &meta; ctx.map = info->map; if (selem) { - sk_storage = rcu_dereference(selem->local_storage); - ctx.sk = sk_storage->owner; + ctx.sk = bpf_sk_storage_selem_get_sk(selem); + if (!ctx.sk) + return SEQ_SKIP; ctx.value = SDATA(selem)->data; } ret = bpf_iter_run_prog(prog, &ctx); + if (ctx.sk) + bpf_sk_storage_map_seq_put_sk(info, ctx.sk); } return ret; @@ -827,6 +899,7 @@ static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) (void)__bpf_sk_storage_map_seq_show(seq, v); else rcu_read_unlock(); + bpf_sk_storage_map_seq_free_sks(seq->private); } static int bpf_iter_init_sk_storage_map(void *priv_data, @@ -846,6 +919,15 @@ static void bpf_iter_fini_sk_storage_map(void *priv_data) bpf_map_put_with_uref(seq_info->map); } +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + if (prog->aux->max_rdwr_access > aux->map->value_size) + return -EACCES; + + return bpf_iter_map_check_value_write(prog, aux->map); +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) @@ -863,12 +945,11 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) goto put_map; - if (prog->aux->max_rdwr_access > map->value_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -899,6 +980,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { .target = "bpf_sk_storage_map", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2,
diff --git a/net/core/filter.c b/net/core/filter.c index 70dc621..b7ac145b 100644 --- a/net/core/filter.c +++ b/net/core/filter.c
@@ -1715,6 +1715,24 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb) skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len); } +/* + * The source of a write through a dynptr may be a slice of the same skb. + * Making the skb writable can move its head: follow the move. + */ +static int bpf_try_make_writable_from(struct sk_buff *skb, u32 write_len, + const void **from) +{ + const void *head = skb->head; + bool in_head = *from >= head && + *from < (const void *)skb_end_pointer(skb); + int err; + + err = bpf_try_make_writable(skb, write_len); + if (!err && in_head) + *from = skb->head + (*from - head); + return err; +} + BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, const void *, from, u32, len, u64, flags) { @@ -1724,14 +1742,15 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, return -EINVAL; if (unlikely(offset > INT_MAX)) return -EFAULT; - if (unlikely(bpf_try_make_writable(skb, offset + len))) + if (unlikely(bpf_try_make_writable_from(skb, offset + len, &from))) return -EFAULT; ptr = skb->data + offset; if (flags & BPF_F_RECOMPUTE_CSUM) __skb_postpull_rcsum(skb, ptr, len, offset); - memcpy(ptr, from, len); + /* through a dynptr the source may be an overlapping slice of the skb */ + memmove(ptr, from, len); if (flags & BPF_F_RECOMPUTE_CSUM) __skb_postpush_rcsum(skb, ptr, len, offset); @@ -3612,6 +3631,14 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff, return -ENOTSUPP; } + /* + * The MSS is lowered by len_diff. Leave the least that the stack lets a + * TCP GSO packet have, or gso_segs can overflow. + */ + if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) && + skb_shinfo(skb)->gso_size < len_diff + TCP_MIN_GSO_SIZE) + return -ENOTSUPP; + ret = skb_cow_head(skb, meta_len + len_diff); if (unlikely(ret < 0)) return ret; @@ -4303,7 +4330,8 @@ BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset, if (!ptr) bpf_xdp_copy_buf(xdp, offset, buf, len, true); else - memcpy(ptr, buf, len); + /* through a dynptr the source may be an overlapping slice */ + memmove(ptr, buf, len); return 0; } @@ -7017,15 +7045,15 @@ static void bpf_update_srh_state(struct sk_buff *skb) { struct seg6_bpf_srh_state *srh_state = this_cpu_ptr(&seg6_bpf_srh_states); - int srhoff = 0; - if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) { - srh_state->srh = NULL; - } else { - srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff); + /* Behind a decapsulation this is a header that nobody has looked at */ + srh_state->srh = seg6_get_srh(skb, 0); + if (srh_state->srh) { srh_state->hdrlen = srh_state->srh->hdrlen << 3; srh_state->valid = true; } + /* it may have been pulled into the head */ + bpf_compute_data_pointers(skb); } BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, @@ -7058,9 +7086,14 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0) return -EBADMSG; - if (!pskb_pull(skb, hdroff)) + /* As long as nothing has been changed, the program goes on */ + if (!pskb_may_pull(skb, hdroff + sizeof(struct ipv6hdr))) { + /* but the head may have moved */ + bpf_update_srh_state(skb); return -EBADMSG; + } + __skb_pull(skb, hdroff); skb_postpull_rcsum(skb, skb_network_header(skb), hdroff); skb_reset_network_header(skb); skb_reset_transport_header(skb); @@ -7074,8 +7107,8 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, return -EBADMSG; err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE, param, param_len); - if (!err) - bpf_update_srh_state(skb); + /* The header can be in place although err says otherwise */ + bpf_update_srh_state(skb); return err; case SEG6_LOCAL_ACTION_END_B6_ENCAP: @@ -7083,8 +7116,8 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, return -EBADMSG; err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6, param, param_len); - if (!err) - bpf_update_srh_state(skb); + /* The header can be in place although err says otherwise */ + bpf_update_srh_state(skb); return err; default: @@ -7600,8 +7633,8 @@ bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, return false; switch (off) { - case offsetof(struct bpf_tcp_sock, bytes_received): - case offsetof(struct bpf_tcp_sock, bytes_acked): + case bpf_ctx_range(struct bpf_tcp_sock, bytes_received): + case bpf_ctx_range(struct bpf_tcp_sock, bytes_acked): return size == sizeof(__u64); default: return size == sizeof(__u32); @@ -8052,17 +8085,14 @@ static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend, return ERR_PTR(-ENOMSG); } -BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, - void *, search_res, u32, len, u64, flags) +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags) { bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN; const u8 *op, *opend, *magic, *search = search_res; u8 search_kind, search_len, copy_len, magic_len; int ret; - if (!is_locked_tcp_sock_ops(bpf_sock)) - return -EOPNOTSUPP; - /* 2 byte is the minimal option len except TCPOPT_NOP and * TCPOPT_EOL which are useless for the bpf prog to learn * and this helper disallow loading them also. @@ -8123,6 +8153,15 @@ BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, return ret; } +BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, + void *, search_res, u32, len, u64, flags) +{ + if (!is_locked_tcp_sock_ops(bpf_sock)) + return -EOPNOTSUPP; + + return __bpf_sock_ops_load_hdr_opt(bpf_sock, search_res, len, flags); +} + static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .func = bpf_sock_ops_load_hdr_opt, .gpl_only = false, @@ -8133,17 +8172,14 @@ static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .arg4_type = ARG_ANYTHING, }; -BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, - const void *, from, u32, len, u64, flags) +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags) { u8 new_kind, new_kind_len, magic_len = 0, *opend; const u8 *op, *new_op, *magic = NULL; struct sk_buff *skb; bool eol; - if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB) - return -EPERM; - if (len < 2 || flags) return -EINVAL; @@ -8201,6 +8237,15 @@ BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, return 0; } +BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, + const void *, from, u32, len, u64, flags) +{ + if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB) + return -EPERM; + + return __bpf_sock_ops_store_hdr_opt(bpf_sock, from, len, flags); +} + static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { .func = bpf_sock_ops_store_hdr_opt, .gpl_only = false, @@ -12402,7 +12447,7 @@ int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset, { if (unlikely(flags)) return -EINVAL; - if (unlikely(bpf_try_make_writable(skb, 0))) + if (unlikely(bpf_try_make_writable_from(skb, 0, &from))) return -EFAULT; memmove(bpf_skb_meta_pointer(skb, offset), from, len); @@ -12878,6 +12923,7 @@ static int __init bpf_kfunc_init(void) ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb_meta); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb_meta); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
diff --git a/net/core/ptp_classifier.c b/net/core/ptp_classifier.c index 598041b..939104e 100644 --- a/net/core/ptp_classifier.c +++ b/net/core/ptp_classifier.c
@@ -2,12 +2,8 @@ /* PTP classifier */ -/* The below program is the bpf_asm (tools/net/) representation of - * the opcode array in the ptp_filter structure. - * - * For convenience, this can easily be altered and reviewed with - * bpf_asm and bpf_dbg, e.g. `./bpf_asm -c prog` where prog is a - * simple file containing the below program: +/* The below program is the classic BPF assembly representation of + * the opcode array in the ptp_filter structure: * * ldh [12] ; load ethertype *
diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 38df842..057e2f3 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c
@@ -1933,6 +1933,15 @@ int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog) return ret; } +static int sock_map_iter_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + if (prog->aux->max_rdonly_access > aux->map->key_size) + return -EACCES; + + return 0; +} + static int sock_map_iter_attach_target(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) @@ -1951,12 +1960,11 @@ static int sock_map_iter_attach_target(struct bpf_prog *prog, map->map_type != BPF_MAP_TYPE_SOCKHASH) goto put_map; - if (prog->aux->max_rdonly_access > map->key_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = sock_map_iter_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -1973,6 +1981,7 @@ static struct bpf_iter_reg sock_map_iter_reg = { .target = "sockmap", .attach_target = sock_map_iter_attach_target, .detach_target = sock_map_iter_detach_target, + .check_prog = sock_map_iter_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2,
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index 69e4baa..e8a0e96 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile
@@ -71,6 +71,7 @@ ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o +obj-$(CONFIG_CGROUP_BPF) += bpf_tcp_ops.o endif ifdef CONFIG_GCOV_PROFILE_NETFILTER
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index c012e2a..7677b03 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c
@@ -226,7 +226,8 @@ int __inet_listen_sk(struct sock *sk, int backlog) if (err) return err; - tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); + tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB); + bpf_tcp_ops_call(listen, sk); } return 0; }
diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 9deed224..2a71dc8 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c
@@ -340,6 +340,22 @@ static struct bpf_struct_ops bpf_tcp_congestion_ops = { .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", .cfi_stubs = &__bpf_ops_tcp_congestion_ops, + /* The struct_ops's function may switch to another struct_ops. + * + * For example, bpf_tcp_cc_x->init() may switch to + * another tcp_cc_y by calling + * setsockopt(TCP_CONGESTION, "tcp_cc_y"). + * During the switch, bpf_struct_ops_put(tcp_cc_x) is called + * and its refcount may reach 0 which then free its + * trampoline image while tcp_cc_x is still running. + * + * A vanilla rcu gp is to wait for all bpf-tcp-cc prog + * to finish. bpf-tcp-cc prog is non sleepable. + * A rcu_tasks gp is to wait for the last few insn + * in the tramopline image to finish before releasing + * the trampoline image. + */ + .free_after_tasks_rcu_gp = true, .owner = THIS_MODULE, };
diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c new file mode 100644 index 0000000..2ba73dd --- /dev/null +++ b/net/ipv4/bpf_tcp_ops.c
@@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <linux/btf_ids.h> +#include <linux/bpf_verifier.h> +#include <linux/filter.h> +#include <net/bpf_sk_storage.h> +#include <net/tcp.h> + +static int timeout_init_stub(struct sock *sk, struct request_sock *req__nullable) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + return ctx->retval; +} + +static int rwnd_init_stub(struct sock *sk, struct request_sock *req__nullable) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + return ctx->retval; +} + +static void active_established_stub(struct sock *sk, struct sk_buff *skb__nullable) +{ +} + +static void passive_established_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void rto_stub(struct sock *sk) +{ +} + +static void rtt_stub(struct sock *sk, long mrtt, u32 srtt) +{ +} + +static void set_state_stub(struct sock *sk, int state) +{ +} + +static void retrans_stub(struct sock *sk, struct sk_buff *skb, int err) +{ +} + +static void connect_stub(struct sock *sk) +{ +} + +static void listen_stub(struct sock *sk) +{ +} + +static void parse_hdr_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void hdr_opt_len_stub(struct sock *sk, struct sk_buff *skb__nullable, + struct request_sock *req__nullable, + struct sk_buff *syn_skb__nullable, + enum tcp_synack_type synack_type, + unsigned int *remaining) +{ +} + +static void write_hdr_opt_stub(struct sock *sk, struct sk_buff *skb, + struct request_sock *req__nullable, + struct sk_buff *syn_skb__nullable, + enum tcp_synack_type synack_type, + u32 opt_off) +{ +} + +static void enqueue_rcvq_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void dequeue_rcvq_stub(struct sock *sk) +{ +} + +static struct bpf_tcp_ops __bpf_tcp_ops = { + .timeout_init = timeout_init_stub, + .rwnd_init = rwnd_init_stub, + .active_established = active_established_stub, + .passive_established = passive_established_stub, + .rto = rto_stub, + .rtt = rtt_stub, + .set_state = set_state_stub, + .retrans = retrans_stub, + .connect = connect_stub, + .listen = listen_stub, + .parse_hdr = parse_hdr_stub, + .hdr_opt_len = hdr_opt_len_stub, + .write_hdr_opt = write_hdr_opt_stub, + .enqueue_rcvq = enqueue_rcvq_stub, + .dequeue_rcvq = dequeue_rcvq_stub, +}; + +BPF_CALL_4(bpf_tcp_ops_store_hdr_opt, void *, ctx, const void *, from, + u32, len, u64, flags) +{ + u64 *args = ctx; + struct sk_buff *skb = (void *)(unsigned long)args[1]; + struct bpf_sock_ops_kern sock_ops = {}; + u32 opt_off = args[5]; + u8 *op, *opend; + + /* + * bpf_tcp_ops does not keep track of the end of the written TCP header + * options, so search for it every time the helper is called. The free + * space is NOP-filled, so a TCPOPT_NOP ends the search rather than being + * skipped as in a normal option walk in sockops. + */ + op = skb->data + opt_off; + opend = skb->data + tcp_hdrlen(skb); + while (op < opend && *op != TCPOPT_NOP) { + if (*op == TCPOPT_EOL || op + 1 >= opend || op[1] < 2) + break; + op += op[1]; + } + + sock_ops.skb = skb; + sock_ops.skb_data_end = op; + sock_ops.remaining_opt_len = opend - op; + + return __bpf_sock_ops_store_hdr_opt(&sock_ops, from, len, flags); +} + +static const struct bpf_func_proto bpf_tcp_ops_store_hdr_opt_proto = { + .func = bpf_tcp_ops_store_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg3_type = ARG_MEM_SIZE, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_4(bpf_tcp_ops_load_hdr_opt, void *, ctx, void *, search_res, + u32, len, u64, flags) +{ + u64 *args = ctx; + struct sk_buff *skb = (void *)(unsigned long)args[1]; + struct bpf_sock_ops_kern sock_ops = {}; + + /* + * No flags supported. In particular BPF_LOAD_HDR_OPT_TCP_SYN, which + * loads from the saved SYN, is not available because bpf_tcp_ops has no + * carrier to track the SYN source across the hooks. + */ + if (flags) + return -EINVAL; + + sock_ops.skb = skb; + sock_ops.skb_data_end = skb->data + tcp_hdrlen(skb); + + return __bpf_sock_ops_load_hdr_opt(&sock_ops, search_res, len, flags); +} + +static const struct bpf_func_proto bpf_tcp_ops_load_hdr_opt_proto = { + .func = bpf_tcp_ops_load_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg3_type = ARG_MEM_SIZE, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_3(bpf_tcp_ops_reserve_hdr_opt, void *, ctx, u32, len, u64, flags) +{ + u64 *args = ctx; + unsigned int *remaining = (void *)(unsigned long)args[5]; + + if (flags || len < 2) + return -EINVAL; + + if (len > *remaining) + return -ENOSPC; + + *remaining -= len; + return 0; +} + +static const struct bpf_func_proto bpf_tcp_ops_reserve_hdr_opt_proto = { + .func = bpf_tcp_ops_reserve_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, +}; + +BPF_CALL_0(bpf_tcp_ops_get_retval) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + /* bpf_get_retval() is only exposed to timeout_init/rwnd_init, which + * always run via bpf_tcp_ops_call_int(). Its run_ctx carries the int + * return value chained across the bpf_tcp_ops attached to the cgroup + * and is this program's saved_run_ctx. + */ + if (WARN_ON_ONCE(!ctx->saved_run_ctx)) + return 0; + + return container_of(ctx->saved_run_ctx, struct bpf_tramp_run_ctx, + run_ctx)->retval; +} + +const struct bpf_func_proto bpf_tcp_ops_get_retval_proto = { + .func = bpf_tcp_ops_get_retval, + .gpl_only = false, + .ret_type = RET_INTEGER, +}; + +static bool is_sockopt_supported(u32 moff) +{ + switch (moff) { + case offsetof(struct bpf_tcp_ops, active_established): + case offsetof(struct bpf_tcp_ops, passive_established): + case offsetof(struct bpf_tcp_ops, rto): + case offsetof(struct bpf_tcp_ops, rtt): + case offsetof(struct bpf_tcp_ops, set_state): + case offsetof(struct bpf_tcp_ops, retrans): + case offsetof(struct bpf_tcp_ops, connect): + case offsetof(struct bpf_tcp_ops, listen): + case offsetof(struct bpf_tcp_ops, parse_hdr): + return true; + } + + return false; +} + +static const struct bpf_func_proto * +get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + u32 moff = prog->aux->attach_st_ops_member_off; + + switch (func_id) { + case BPF_FUNC_sk_storage_get: + return &bpf_sk_storage_get_proto; + case BPF_FUNC_sk_storage_delete: + return &bpf_sk_storage_delete_proto; + case BPF_FUNC_setsockopt: + if (is_sockopt_supported(moff)) + return &bpf_sk_setsockopt_proto; + return NULL; + case BPF_FUNC_getsockopt: + if (is_sockopt_supported(moff)) + return &bpf_sk_getsockopt_proto; + return NULL; + case BPF_FUNC_get_retval: + if (moff == offsetof(struct bpf_tcp_ops, timeout_init) || + moff == offsetof(struct bpf_tcp_ops, rwnd_init)) + return &bpf_tcp_ops_get_retval_proto; + return NULL; + case BPF_FUNC_reserve_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, hdr_opt_len)) + return &bpf_tcp_ops_reserve_hdr_opt_proto; + return NULL; + case BPF_FUNC_load_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, parse_hdr) || + moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) + return &bpf_tcp_ops_load_hdr_opt_proto; + return NULL; + case BPF_FUNC_store_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) + return &bpf_tcp_ops_store_hdr_opt_proto; + return NULL; + default: + return bpf_base_func_proto(func_id, prog); + } +} + +static bool is_valid_access(int off, int size, enum bpf_access_type type, + const struct bpf_prog *prog, struct bpf_insn_access_aux *info) +{ + if (!bpf_tracing_btf_ctx_access(off, size, type, prog, info)) + return false; + + if (base_type(info->reg_type) == PTR_TO_BTF_ID && + !bpf_type_has_unsafe_modifiers(info->reg_type) && + info->btf_id == btf_sock_ids[BTF_SOCK_TYPE_SOCK]) + /* promote it to tcp_sock */ + info->btf_id = btf_sock_ids[BTF_SOCK_TYPE_TCP]; + + return true; +} + +static int bpf_tcp_ops_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + return 0; +} + +static int bpf_tcp_ops_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + if (prog->sleepable) + return -EINVAL; + + return 0; +} + +static int bpf_tcp_ops_init(struct btf *btf) +{ + return 0; +} + +static int bpf_tcp_ops_validate(void *kdata) +{ + return 0; +} + +static const struct bpf_verifier_ops bpf_tcp_ops_verifier = { + .get_func_proto = get_func_proto, + .is_valid_access = is_valid_access, +}; + +static struct bpf_struct_ops bpf_tcp_ops = { + .verifier_ops = &bpf_tcp_ops_verifier, + .init_member = bpf_tcp_ops_init_member, + .check_member = bpf_tcp_ops_check_member, + .init = bpf_tcp_ops_init, + .validate = bpf_tcp_ops_validate, + .name = "bpf_tcp_ops", + .cgroup_atype = CGROUP_TCP_SOCK_OPS, + .cfi_stubs = &__bpf_tcp_ops, + .owner = THIS_MODULE, +}; + +__bpf_kfunc_start_defs(); + +__bpf_kfunc int bpf_tcp_ops_set_flags(struct tcp_sock *tp, u32 enable, u32 disable) +{ + u32 old, new; + + if ((enable & disable) || (enable | disable) & ~BPF_TCP_OPS_FLAG_ALL) + return -EINVAL; + + if (sk_is_mptcp((struct sock *)tp) && (enable & BPF_TCP_OPS_FLAG_RCVQ)) + return -EOPNOTSUPP; + + old = READ_ONCE(tp->bpf_tcp_ops_flags); + + do { + new = (old | enable) & ~disable; + if (new == old) + break; + } while (!try_cmpxchg(&tp->bpf_tcp_ops_flags, &old, new)); + + return 0; +} + +__bpf_kfunc int bpf_tcp_ops_set_rcvlowat(struct sock *sk, int rcvlowat, + const struct bpf_prog_aux *aux) +{ + u32 moff = aux->attach_st_ops_member_off; + bool wakeup = false; + + if (moff == offsetof(struct bpf_tcp_ops, dequeue_rcvq)) + wakeup = true; + + if (rcvlowat < 0) + rcvlowat = INT_MAX; + + return __tcp_set_rcvlowat(sk, rcvlowat, wakeup); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(bpf_tcp_ops_set_flags_kfunc_set) +BTF_ID_FLAGS(func, bpf_tcp_ops_set_flags) +BTF_KFUNCS_END(bpf_tcp_ops_set_flags_kfunc_set) + +BTF_KFUNCS_START(bpf_tcp_ops_set_rcvlowat_kfunc_set) +BTF_ID_FLAGS(func, bpf_tcp_ops_set_rcvlowat, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(bpf_tcp_ops_set_rcvlowat_kfunc_set) + +static int bpf_tcp_ops_set_flags_kfunc_filter(const struct bpf_prog *prog, + u32 kfunc_id) +{ + if (!btf_id_set8_contains(&bpf_tcp_ops_set_flags_kfunc_set, kfunc_id)) + return 0; + + if (prog->type == BPF_PROG_TYPE_STRUCT_OPS && + prog->aux->st_ops == &bpf_tcp_ops) + return 0; + + if (prog->type == BPF_PROG_TYPE_CGROUP_SOCKOPT || + prog->type == BPF_PROG_TYPE_CGROUP_SKB) + return 0; + + return -EACCES; +} + +static const struct btf_kfunc_id_set bpf_tcp_ops_set_flags_kfunc_id_set = { + .owner = THIS_MODULE, + .set = &bpf_tcp_ops_set_flags_kfunc_set, + .filter = bpf_tcp_ops_set_flags_kfunc_filter, +}; + +static int bpf_tcp_ops_set_rcvlowat_kfunc_filter(const struct bpf_prog *prog, + u32 kfunc_id) +{ + u32 moff; + + if (!btf_id_set8_contains(&bpf_tcp_ops_set_rcvlowat_kfunc_set, kfunc_id)) + return 0; + + if (prog->aux->st_ops != &bpf_tcp_ops) + return -EACCES; + + moff = prog->aux->attach_st_ops_member_off; + if (moff != offsetof(struct bpf_tcp_ops, enqueue_rcvq) && + moff != offsetof(struct bpf_tcp_ops, dequeue_rcvq)) + return -EACCES; + + return 0; +} + +static const struct btf_kfunc_id_set bpf_tcp_ops_set_rcvlowat_kfunc_id_set = { + .owner = THIS_MODULE, + .set = &bpf_tcp_ops_set_rcvlowat_kfunc_set, + .filter = bpf_tcp_ops_set_rcvlowat_kfunc_filter, +}; + +static int __init __bpf_tcp_ops_init(void) +{ + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_tcp_ops_set_flags_kfunc_id_set); + /* BPF_PROG_TYPE_CGROUP_{SOCKOPT,SKB} share BTF_KFUNC_HOOK_CGROUP. */ + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCKOPT, + &bpf_tcp_ops_set_flags_kfunc_id_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_tcp_ops_set_rcvlowat_kfunc_id_set); + ret = ret ?: register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops); + + return ret; +} + +late_initcall(__bpf_tcp_ops_init);
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 3fd4776..59e5216 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c
@@ -1612,6 +1612,8 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied) "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); __tcp_cleanup_rbuf(sk, copied); + + bpf_tcp_ops_call_flag(dequeue_rcvq, RCVQ, sk); } static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) @@ -1828,8 +1830,7 @@ int tcp_peek_len(struct socket *sock) return tcp_inq(sock->sk); } -/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ -int tcp_set_rcvlowat(struct sock *sk, int val) +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup) { struct tcp_sock *tp = tcp_sk(sk); int space, cap; @@ -1842,7 +1843,8 @@ int tcp_set_rcvlowat(struct sock *sk, int val) WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); /* Check if we need to signal EPOLLIN right now */ - tcp_data_ready(sk); + if (wakeup) + tcp_data_ready(sk); if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) return 0; @@ -1857,6 +1859,12 @@ int tcp_set_rcvlowat(struct sock *sk, int val) return 0; } +/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ +int tcp_set_rcvlowat(struct sock *sk, int val) +{ + return __tcp_set_rcvlowat(sk, val, true); +} + void tcp_set_rcvbuf(struct sock *sk, int val) { tcp_set_window_clamp(sk, tcp_win_from_space(sk, val)); @@ -2986,8 +2994,8 @@ void tcp_set_state(struct sock *sk, int state) */ BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); + bpf_tcp_ops_call(set_state, sk, state); switch (state) { case TCP_ESTABLISHED: @@ -5208,6 +5216,9 @@ static void __init tcp_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio); +#ifdef CONFIG_BPF + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, bpf_tcp_ops_flags); +#endif /* RX read-mostly hotpath cache lines */ CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq);
diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index c2b024a..8cf63a9 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c
@@ -108,7 +108,6 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, off = sge->offset; page = sg_page(sge); - tcp_rate_check_app_limited(sk); retry: msghdr.msg_flags = flags | MSG_SPLICE_PAGES; has_tx_ulp = tls_sw_has_ctx_tx(sk);
diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 471c78b..6dfe403 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c
@@ -281,6 +281,8 @@ void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->seq++; TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN; + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; tcp_add_receive_queue(sk, skb); tp->syn_data_acked = 1;
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 1ddce9c..e1f2d97 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c
@@ -146,14 +146,16 @@ EXPORT_SYMBOL_GPL(clean_acked_data_flush); #ifdef CONFIG_CGROUP_BPF static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) { - bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown && - BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG); - bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG); struct bpf_sock_ops_kern sock_ops; + const struct tcp_sock *tp; - if (likely(!unknown_opt && !parse_all_opt)) + if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + return; + + tp = tcp_sk(sk); + if (!(tp->rx_opt.saw_unknown && + BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG)) && + !BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG)) return; /* The skb will be handled in the @@ -176,7 +178,7 @@ static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) sock_ops.sk = sk; bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb)); - BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); } static void bpf_skops_established(struct sock *sk, int bpf_op, @@ -184,6 +186,9 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, { struct bpf_sock_ops_kern sock_ops; + if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + return; + sock_owned_by_me(sk); memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); @@ -195,7 +200,7 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, if (skb) bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb)); - BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); } #else static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) @@ -208,6 +213,28 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, } #endif +static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb) +{ + const struct tcp_sock *tp; + + if (!cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) + return; + + switch (sk->sk_state) { + case TCP_SYN_RECV: + case TCP_SYN_SENT: + case TCP_LISTEN: + return; + } + + tp = tcp_sk(sk); + + if ((tp->rx_opt.saw_unknown && + BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_UNKNOWN)) || + BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_ALL)) + __bpf_tcp_ops_call(parse_hdr, sk, skb); +} + static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb, unsigned int len) { @@ -5338,6 +5365,8 @@ static void tcp_ofo_queue(struct sock *sk) continue; } + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + tail = skb_peek_tail(&sk->sk_receive_queue); eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen); tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq); @@ -5541,6 +5570,8 @@ static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int eaten; struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue); + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + eaten = (tail && tcp_try_coalesce(sk, tail, skb, fragstolen)) ? 1 : 0; @@ -6480,6 +6511,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, pass: bpf_skops_parse_hdr(sk, skb); + bpf_tcp_ops_parse_hdr(sk, skb); return true; @@ -6744,6 +6776,10 @@ void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb) tp->snd_cwnd_stamp = tcp_jiffies32; bpf_skops_established(sk, bpf_op, skb); + if (bpf_op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB) + bpf_tcp_ops_call(active_established, sk, skb); + else + bpf_tcp_ops_call(passive_established, sk, skb); /* Initialize congestion control unless BPF initialized it already: */ if (!icsk->icsk_ca_initialized) tcp_init_congestion_control(sk);
diff --git a/net/ipv4/tcp_nv.c b/net/ipv4/tcp_nv.c index f345897..7b0dae2 100644 --- a/net/ipv4/tcp_nv.c +++ b/net/ipv4/tcp_nv.c
@@ -146,7 +146,7 @@ static void tcpnv_init(struct sock *sk) * within a datacenter, where we have reasonable estimates of * RTTs */ - base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT, 0, NULL); + base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT); if (base_rtt > 0) { ca->nv_base_rtt = base_rtt; ca->nv_lower_bound_rtt = (base_rtt * 205) >> 8; /* 80% */
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index c886520..df3aaba 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c
@@ -476,8 +476,8 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, struct bpf_sock_ops_kern sock_ops; int err; - if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) || + if (!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG) || !remaining) return remaining; @@ -518,7 +518,7 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, if (skb) bpf_skops_init_skb(&sock_ops, skb, 0); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); if (err || sock_ops.remaining_opt_len == remaining) return remaining; @@ -536,43 +536,54 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, enum tcp_synack_type synack_type, struct tcp_out_options *opts) { - u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len; - struct bpf_sock_ops_kern sock_ops; - int err; + u8 first_opt_off, nr_written = 0, max_opt_len = opts->bpf_opt_len; if (likely(!max_opt_len)) return; - memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); - - sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; - - if (req) { - sock_ops.sk = (struct sock *)req; - sock_ops.syn_skb = syn_skb; - } else { - sock_owned_by_me(sk); - - sock_ops.is_fullsock = 1; - sock_ops.is_locked_tcp_sock = 1; - sock_ops.sk = sk; - } - - sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); - sock_ops.remaining_opt_len = max_opt_len; first_opt_off = tcp_hdrlen(skb) - max_opt_len; - bpf_skops_init_skb(&sock_ops, skb, first_opt_off); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) { + struct bpf_sock_ops_kern sock_ops; + int err; - if (err) - nr_written = 0; - else - nr_written = max_opt_len - sock_ops.remaining_opt_len; + memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); + + sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; + + if (req) { + sock_ops.sk = (struct sock *)req; + sock_ops.syn_skb = syn_skb; + } else { + sock_owned_by_me(sk); + + sock_ops.is_fullsock = 1; + sock_ops.is_locked_tcp_sock = 1; + sock_ops.sk = sk; + } + + sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); + sock_ops.remaining_opt_len = max_opt_len; + bpf_skops_init_skb(&sock_ops, skb, first_opt_off); + + err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + if (!err) + nr_written = max_opt_len - sock_ops.remaining_opt_len; + } if (nr_written < max_opt_len) memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP, max_opt_len - nr_written); + + /* + * bpf_tcp_ops portion is NOP-filled (everything past the sockops + * writer's bytes). The writer finds the append point by scanning from + * first_opt_off + nr_written to the first NOP. + */ + bpf_tcp_ops_call_flag(write_hdr_opt, WRITE_HDR_OPT, sk, skb, req, + syn_skb, synack_type, first_opt_off + nr_written); } #else static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, @@ -594,6 +605,53 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, } #endif +static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, + struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + struct tcp_out_options *opts, + u32 remaining) +{ + unsigned int remaining_out = remaining, reserved; + + if (!BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), WRITE_HDR_OPT) || + !remaining) + return remaining; + + /* bpf_tcp_ops_reserve_hdr_opt() reserves space via remaining_out */ + __bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out); + + reserved = remaining - remaining_out; + if (!reserved) + return remaining; + + /* round up to 4 bytes */ + reserved = (reserved + 3) & ~3; + + opts->bpf_opt_len += reserved; + return remaining - reserved; +} + +static __always_inline u32 tcp_bpf_hdr_opt_len(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, + struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + struct tcp_out_options *opts, + u32 remaining) +{ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + remaining = bpf_skops_hdr_opt_len(sk, skb, req, syn_skb, + synack_type, opts, + remaining); + + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) + remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, req, syn_skb, + synack_type, opts, + remaining); + + return remaining; +} + static __be32 *process_tcp_ao_options(struct tcp_sock *tp, const struct tcp_request_sock *tcprsk, struct tcp_out_options *opts, @@ -1051,8 +1109,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, remaining -= tcp_options_fit_accecn(opts, 0, remaining); } - remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); + remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1139,8 +1197,8 @@ static unsigned int tcp_synack_options(const struct sock *sk, remaining -= tcp_options_fit_accecn(opts, 0, remaining); } - remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, - synack_type, opts, remaining); + remaining = tcp_bpf_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, + synack_type, opts, remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1153,10 +1211,12 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb struct tcp_key *key) { struct tcp_sock *tp = tcp_sk(sk); - unsigned int size = 0; unsigned int eff_sacks; + unsigned int remaining; + unsigned int size = 0; opts->options = 0; + opts->bpf_opt_len = 0; /* Better than switch (key.type) as it has static branches */ if (tcp_key_is_md5(key)) { @@ -1183,10 +1243,10 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb * left. */ if (sk_is_mptcp(sk)) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; bool has_ts = opts->options & OPTION_TS; int opt_size; + remaining = MAX_TCP_OPTION_SPACE - size; opts->mptcp.drop_ts = 0; opt_size = mptcp_established_options(sk, skb, remaining, has_ts, @@ -1203,7 +1263,8 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack; if (unlikely(eff_sacks)) { - const unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + remaining = MAX_TCP_OPTION_SPACE - size; + if (likely(remaining >= TCPOLEN_SACK_BASE_ALIGNED + TCPOLEN_SACK_PERBLOCK)) { opts->num_sack_blocks = @@ -1234,15 +1295,10 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb } } - if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp, - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + MAX_TCP_OPTION_SPACE - size); - remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); - - size = MAX_TCP_OPTION_SPACE - remaining; - } + size = MAX_TCP_OPTION_SPACE - remaining; return size; } @@ -3683,9 +3739,9 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); } - if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG)) - tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB, - TCP_SKB_CB(skb)->seq, segs, err); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RETRANS_CB, + TCP_SKB_CB(skb)->seq, segs, err); + bpf_tcp_ops_call(retrans, sk, skb, err); if (unlikely(err) && err != -EBUSY) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs); @@ -4312,7 +4368,8 @@ int tcp_connect(struct sock *sk) struct sk_buff *buff; int err; - tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL); + tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB); + bpf_tcp_ops_call(connect, sk); #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO) /* Has to be checked late, after setting daddr/saddr/ops.
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index e56eae4..00debcb 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c
@@ -286,10 +286,9 @@ static int tcp_write_timeout(struct sock *sk) tcp_fastopen_active_detect_blackhole(sk, expired); mptcp_active_detect_blackhole(sk, expired); - if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RTO_CB_FLAG)) - tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RTO_CB, - icsk->icsk_retransmits, - icsk->icsk_rto, (int)expired); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTO_CB, + icsk->icsk_retransmits, icsk->icsk_rto, (int)expired); + bpf_tcp_ops_call(rto, sk); if (expired) { /* Has it gone just too far? */
diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c index 098ca02..5691c13 100644 --- a/net/sched/bpf_qdisc.c +++ b/net/sched/bpf_qdisc.c
@@ -280,7 +280,6 @@ BTF_KFUNCS_START(qdisc_kfunc_ids) BTF_ID_FLAGS(func, bpf_skb_get_hash) BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE) BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_dynptr_from_skb) BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule) BTF_ID_FLAGS(func, bpf_qdisc_init_prologue) BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue) @@ -290,7 +289,6 @@ BTF_KFUNCS_END(qdisc_kfunc_ids) BTF_SET_START(qdisc_common_kfunc_set) BTF_ID(func, bpf_skb_get_hash) BTF_ID(func, bpf_kfree_skb) -BTF_ID(func, bpf_dynptr_from_skb) BTF_SET_END(qdisc_common_kfunc_set) BTF_SET_START(qdisc_enqueue_kfunc_set)
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 95a4fa1..c28e650 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile
@@ -342,6 +342,7 @@ $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/xdp_sample_shared.h @echo " CLANG-BPF " $@ $(Q)$(CLANG) -g -O2 --target=bpf -D__TARGET_ARCH_$(SRCARCH) \ + -fms-extensions -Wno-microsoft-anon-tag \ -Wno-compare-distinct-pointer-types -I$(srctree)/include \ -I$(srctree)/samples/bpf -I$(srctree)/tools/include \ -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \
diff --git a/samples/bpf/hash_func01.h b/samples/bpf/hash_func01.h deleted file mode 100644 index 3825581..0000000 --- a/samples/bpf/hash_func01.h +++ /dev/null
@@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 - * - * Based on Paul Hsieh's (LGPG 2.1) hash function - * From: http://www.azillionmonkeys.com/qed/hash.html - */ - -#define get16bits(d) (*((const __u16 *) (d))) - -static __always_inline -__u32 SuperFastHash (const char *data, int len, __u32 initval) { - __u32 hash = initval; - __u32 tmp; - int rem; - - if (len <= 0 || data == NULL) return 0; - - rem = len & 3; - len >>= 2; - - /* Main loop */ -#pragma clang loop unroll(full) - for (;len > 0; len--) { - hash += get16bits (data); - tmp = (get16bits (data+2) << 11) ^ hash; - hash = (hash << 16) ^ tmp; - data += 2*sizeof (__u16); - hash += hash >> 11; - } - - /* Handle end cases */ - switch (rem) { - case 3: hash += get16bits (data); - hash ^= hash << 16; - hash ^= ((signed char)data[sizeof (__u16)]) << 18; - hash += hash >> 11; - break; - case 2: hash += get16bits (data); - hash ^= hash << 11; - hash += hash >> 17; - break; - case 1: hash += (signed char)*data; - hash ^= hash << 10; - hash += hash >> 1; - } - - /* Force "avalanching" of final 127 bits */ - hash ^= hash << 3; - hash += hash >> 5; - hash ^= hash << 4; - hash += hash >> 17; - hash ^= hash << 25; - hash += hash >> 6; - - return hash; -}
diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf index a181298..9801234 100644 --- a/scripts/Makefile.btf +++ b/scripts/Makefile.btf
@@ -22,7 +22,14 @@ pahole-flags-$(CONFIG_PAHOLE_HAS_LANG_EXCLUDE) += --lang_exclude=rust +btf-inline := $(CONFIG_DEBUG_INFO_BTF_INLINE) +ifneq ($(btf-inline),) +btf-inline-feat := inline +pahole-flags-$(call test-ge, $(pahole-ver), 132) += --btf_features=$(btf-inline-feat) +endif + export PAHOLE_FLAGS := $(pahole-flags-y) +export BTF_INLINE := $(btf-inline) resolve-btfids-flags-y := resolve-btfids-flags-$(CONFIG_WERROR) += --fatal_warnings
diff --git a/scripts/gen-btf.sh b/scripts/gen-btf.sh index 8ca96eb..b17f118 100755 --- a/scripts/gen-btf.sh +++ b/scripts/gen-btf.sh
@@ -68,6 +68,7 @@ gen_btf_data() { + inline=${BTF_INLINE:+--inline} btf1="${ELF_FILE}.BTF.1" ${PAHOLE} -J ${PAHOLE_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ @@ -76,6 +77,7 @@ ${RESOLVE_BTFIDS} ${RESOLVE_BTFIDS_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ + ${inline} \ --btf ${btf1} "${ELF_FILE}" } @@ -83,14 +85,28 @@ { btf_data=${ELF_FILE}.btf.o - # Create ${btf_data} which contains just .BTF section but no symbols. Add + # Create ${btf_data} which contains just BTF sections but no symbols. Add # SHF_ALLOC because .BTF will be part of the vmlinux image. --strip-all # deletes all symbols including __start_BTF and __stop_BTF, which will # be redefined in the linker script. echo "" | ${CC} ${CLANG_FLAGS} ${KBUILD_CPPFLAGS} ${KBUILD_CFLAGS} -fno-lto -c -x c -o ${btf_data} - ${OBJCOPY} --add-section .BTF=${ELF_FILE}.BTF \ --set-section-flags .BTF=alloc,readonly ${btf_data} - ${OBJCOPY} --only-section=.BTF --strip-all ${btf_data} + ONLY_SEC="--only-section=.BTF" + btf_inline=${ELF_FILE}.BTF.inline + if [ "${BTF_INLINE}" = "m" ]; then + if [ -f "${btf_inline}" ]; then + cp "${btf_inline}" "${objtree}/vmlinux.BTF.inline" + else + rm -f "${objtree}/vmlinux.BTF.inline" + fi + fi + if [ "${BTF_INLINE}" = "y" ] && [ -f "${btf_inline}" ]; then + ${OBJCOPY} --add-section .BTF.inline=${btf_inline} \ + --set-section-flags .BTF.inline=alloc,readonly ${btf_data} + ONLY_SEC="${ONLY_SEC} --only-section=.BTF.inline" + fi + ${OBJCOPY} ${ONLY_SEC} --strip-all ${btf_data} # Change e_type to ET_REL so that it can be used to link final vmlinux. # GNU ld 2.35+ and lld do not allow an ET_EXEC input. @@ -111,6 +127,15 @@ if [ -f "${btf_base}" ]; then ${OBJCOPY} --add-section .BTF.base=${btf_base} ${ELF_FILE} fi + btf_inline=${ELF_FILE}.BTF.inline + case "${ELF_FILE}" in + */btf_vmlinux_inline.ko) + btf_inline=${BTF_BASE}.BTF.inline + ;; + esac + if [ -n "${BTF_INLINE}" ] && [ -f "${btf_inline}" ]; then + ${OBJCOPY} --add-section .BTF.inline=${btf_inline} ${ELF_FILE} + fi btf_ids="${ELF_FILE}.BTF_ids" if [ -f "${btf_ids}" ]; then ${RESOLVE_BTFIDS} --patch_btfids ${btf_ids} ${ELF_FILE} @@ -121,6 +146,8 @@ { rm -f "${ELF_FILE}.BTF.1" rm -f "${ELF_FILE}.BTF" + rm -f "${ELF_FILE}.BTF.inline" + rm -f "${ELF_FILE}.BTF.inline.link" if [ "${BTFGEN_MODE}" = "module" ]; then rm -f "${ELF_FILE}.BTF.base" rm -f "${ELF_FILE}.BTF_ids"
diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index c520741..d5710e9 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh
@@ -295,7 +295,13 @@ if is_enabled CONFIG_DEBUG_INFO_BTF; then info BTFIDS ${VMLINUX} - ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX} + if [ "${BTF_INLINE}" = "m" ]; then + ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} \ + --btf_link .BTF.inline:btf_vmlinux_inline:vmlinux.BTF.inline \ + ${VMLINUX} + else + ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX} + fi fi mksysmap "${VMLINUX}" System.map
diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 46e8097..8f0933c 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec
@@ -68,13 +68,13 @@ %undefine _debugsource_packages %undefine _debuginfo_subpackages -# Preserve .BTF and .BTF.base sections in kernel modules during debuginfo -# stripping. find-debuginfo.sh uses eu-strip which removes non-allocated ELF -# sections like .BTF by default. .BTF.base is required for BTF distillation -# support; without it, module BTF validation fails. +# Preserve .BTF, .BTF.base and .BTF.inline sections in kernel modules during +# debuginfo stripping. find-debuginfo.sh uses eu-strip which removes +# non-allocated ELF sections like .BTF by default. .BTF.base is required for +# BTF distillation support; without it, module BTF validation fails. %global with_keep_section %(%{__find_debuginfo} --help 2>&1 | grep -c keep-section) %if %{with_keep_section} -%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base +%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base --keep-section .BTF.inline %else %global _find_debuginfo_opts -r %endif
diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c index 7b98f5d..8f8c3de 100644 --- a/security/bpf/hooks.c +++ b/security/bpf/hooks.c
@@ -33,6 +33,7 @@ static int __init bpf_lsm_init(void) struct lsm_blob_sizes bpf_lsm_blob_sizes __ro_after_init = { .lbs_inode = sizeof(struct bpf_storage_blob), + .lbs_xattr_count = BPF_LSM_INODE_INIT_XATTRS, }; DEFINE_LSM(bpf) = {
diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index 1a51a36..cf638bf 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c
@@ -22,6 +22,9 @@ /* Session keyring create vs join semaphore */ static DEFINE_MUTEX(key_session_mutex); +/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */ +static struct key *bpf_keyring __ro_after_init; + /* The root user's tracking struct */ struct key_user root_key_user = { .usage = REFCOUNT_INIT(3), @@ -589,6 +592,20 @@ bool lookup_user_key_possessed(const struct key *key, return key == match_data->raw_data; } +/** + * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING + * @keyring: The keyring to publish + * + * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING + * special key ID, so that provisioning it does not require scraping its + * serial out of /proc/keys first. Called once, from an initcall, and never + * undone. + */ +void key_register_bpf_keyring(struct key *keyring) +{ + bpf_keyring = keyring; +} + /* * Look up a key ID given us by userspace with a given permissions mask to get * the key it refers to. @@ -740,6 +757,14 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, key_ref = make_key_ref(key, 1); break; + case KEY_SPEC_BPF_KEYRING: + key = bpf_keyring; + if (!key) + goto error; + __key_get(key); + key_ref = make_key_ref(key, 0); + break; + default: key_ref = ERR_PTR(-EINVAL); if (id < 1)
diff --git a/tools/arch/arm64/include/asm/barrier.h b/tools/arch/arm64/include/asm/barrier.h index 3b9b413..abdc64f 100644 --- a/tools/arch/arm64/include/asm/barrier.h +++ b/tools/arch/arm64/include/asm/barrier.h
@@ -24,6 +24,10 @@ #define smp_wmb() asm volatile("dmb ishst" ::: "memory") #define smp_rmb() asm volatile("dmb ishld" ::: "memory") +/* DMA barriers use outer-shareable (osh) for device visibility */ +#define dma_rmb() asm volatile("dmb oshld" ::: "memory") +#define dma_wmb() asm volatile("dmb oshst" ::: "memory") + #define smp_store_release(p, v) \ do { \ union { typeof(*p) __val; char __c[1]; } __u = \
diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index 1c13bfa..9998252 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h
@@ -97,6 +97,11 @@ struct kvm_regs { #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) #define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) +/* Supported VGICv5 address types */ +#define KVM_VGIC_V5_ADDR_TYPE_IRS 6 + +#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ @@ -417,6 +422,16 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 +#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 +#define KVM_DEV_ARM_VGIC_GRP_IST 11 + +struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; +}; + #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
diff --git a/tools/arch/x86/include/asm/barrier.h b/tools/arch/x86/include/asm/barrier.h index 0adf295..0b51431 100644 --- a/tools/arch/x86/include/asm/barrier.h +++ b/tools/arch/x86/include/asm/barrier.h
@@ -43,4 +43,9 @@ do { \ ___p1; \ }) #endif /* defined(__x86_64__) */ + +/* x86 is DMA-coherent so DMA barriers are just compiler barriers */ +#define dma_rmb() barrier() +#define dma_wmb() barrier() + #endif /* _TOOLS_LINUX_ASM_X86_BARRIER_H */
diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h index 1585ec8..00ce265 100644 --- a/tools/arch/x86/include/uapi/asm/kvm.h +++ b/tools/arch/x86/include/uapi/asm/kvm.h
@@ -45,9 +45,7 @@ #define __KVM_HAVE_PIT #define __KVM_HAVE_IOAPIC #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_MSI #define __KVM_HAVE_USER_NMI -#define __KVM_HAVE_MSIX #define __KVM_HAVE_MCE #define __KVM_HAVE_PIT_STATE2 #define __KVM_HAVE_XEN_HVM
diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c index df05340..6f476e3 100644 --- a/tools/bootconfig/main.c +++ b/tools/bootconfig/main.c
@@ -17,9 +17,38 @@ #define pr_err(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__) -/* Bootconfig footer is [size][csum][BOOTCONFIG_MAGIC]. */ -#define BOOTCONFIG_FOOTER_SIZE \ - (sizeof(uint32_t) * 2 + BOOTCONFIG_MAGIC_LEN) +static void show_xbc_error(const char *data, const char *msg, int pos) +{ + int lin = 1, col, i; + + if (pos < 0) { + pr_err("Error: %s.\n", msg); + return; + } + + /* Note that pos starts from 0 but lin and col should start from 1. */ + col = pos + 1; + for (i = 0; i < pos; i++) { + if (data[i] == '\n') { + lin++; + col = pos - i; + } + } + pr_err("Parse Error: %s at %d:%d\n", msg, lin, col); + +} + +static int init_xbc_with_error(char *buf, int len) +{ + const char *msg; + int ret, pos; + + ret = xbc_init(buf, len, &msg, &pos); + if (ret < 0) + show_xbc_error(buf, msg, pos); + + return ret; +} static int xbc_show_value(struct xbc_node *node, bool semicolon) { @@ -134,8 +163,6 @@ static void xbc_show_list(void) } } -#define PAGE_SIZE 4096 - static int load_xbc_fd(int fd, char **buf, int size) { int ret; @@ -197,7 +224,6 @@ static int load_xbc_from_initrd(int fd, char **buf) int ret; uint32_t size = 0, csum = 0, rcsum; char magic[BOOTCONFIG_MAGIC_LEN]; - const char *msg; ret = fstat(fd, &stat); if (ret < 0) @@ -249,52 +275,9 @@ static int load_xbc_from_initrd(int fd, char **buf) return -EINVAL; } - ret = xbc_init(*buf, size, &msg, NULL); - /* Wrong data */ - if (ret < 0) { - pr_err("parse error: %s.\n", msg); - return ret; - } + ret = init_xbc_with_error(*buf, size); - return size; -} - -static void show_xbc_error(const char *data, const char *msg, int pos) -{ - int lin = 1, col, i; - - if (pos < 0) { - pr_err("Error: %s.\n", msg); - return; - } - - /* Note that pos starts from 0 but lin and col should start from 1. */ - col = pos + 1; - for (i = 0; i < pos; i++) { - if (data[i] == '\n') { - lin++; - col = pos - i; - } - } - pr_err("Parse Error: %s at %d:%d\n", msg, lin, col); - -} - -static int init_xbc_with_error(char *buf, int len) -{ - char *copy = strdup(buf); - const char *msg; - int ret, pos; - - if (!copy) - return -ENOMEM; - - ret = xbc_init(buf, len, &msg, &pos); - if (ret < 0) - show_xbc_error(copy, msg, pos); - free(copy); - - return ret; + return ret < 0 ? ret : size; } static int show_xbc_kernel_cmdline(void) @@ -423,9 +406,8 @@ static int apply_xbc(const char *path, const char *xbc_path) char *buf, *data; size_t total_size; struct stat stat; - const char *msg; uint32_t size, csum; - int pos, pad; + int pad; int ret, fd; ret = load_xbc_file(xbc_path, &buf); @@ -433,9 +415,18 @@ static int apply_xbc(const char *path, const char *xbc_path) pr_err("Failed to load %s : %d\n", xbc_path, ret); return ret; } - size = strlen(buf) + 1; + size = ret; + if (size == 0 || buf[size - 1] != '\0') + size++; csum = xbc_calc_checksum(buf, size); + /* Verify the data format */ + ret = init_xbc_with_error(buf, size); + if (ret < 0) { + free(buf); + return ret; + } + /* Backup the bootconfig data */ data = calloc(size + BOOTCONFIG_ALIGN + BOOTCONFIG_FOOTER_SIZE, 1); if (!data) { @@ -444,15 +435,6 @@ static int apply_xbc(const char *path, const char *xbc_path) } memcpy(data, buf, size); - /* Check the data format */ - ret = xbc_init(buf, size, &msg, &pos); - if (ret < 0) { - show_xbc_error(data, msg, pos); - free(data); - free(buf); - - return ret; - } printf("Apply %s to %s\n", xbc_path, path); xbc_get_info(&ret, NULL); printf("\tNumber of nodes: %d\n", ret);
diff --git a/tools/bootconfig/test-bootconfig.sh b/tools/bootconfig/test-bootconfig.sh index fc69f81..530ce7e 100755 --- a/tools/bootconfig/test-bootconfig.sh +++ b/tools/bootconfig/test-bootconfig.sh
@@ -180,6 +180,18 @@ $BOOTCONF -a $TEMPCONF $INITRD 2> $OUTFILE xpass grep -q "1:1" $OUTFILE +echo "Intermediate null character test" +printf "key = value\n\0extra = data\n" > $TEMPCONF +xfail $BOOTCONF -a $TEMPCONF $INITRD +$BOOTCONF -a $TEMPCONF $INITRD 2> $OUTFILE +xpass grep -q "Unexpected" $OUTFILE + +echo "Trailing null character test" +printf "key = value\n\0" > $TEMPCONF +xpass $BOOTCONF -a $TEMPCONF $INITRD +$BOOTCONF $INITRD > $OUTFILE +xpass grep -q "value" $OUTFILE + echo "=== expected failure cases ===" for i in samples/bad-* ; do xfail $BOOTCONF -a $i $INITRD
diff --git a/tools/bpf/.gitignore b/tools/bpf/.gitignore deleted file mode 100644 index cf53342..0000000 --- a/tools/bpf/.gitignore +++ /dev/null
@@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -FEATURE-DUMP.bpf -feature -bpf_asm -bpf_dbg -bpf_exp.yacc.* -bpf_jit_disasm
diff --git a/tools/bpf/Makefile b/tools/bpf/Makefile index 9c19e81..a41360e 100644 --- a/tools/bpf/Makefile +++ b/tools/bpf/Makefile
@@ -1,108 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 include ../scripts/Makefile.include -prefix ?= /usr/local - -LEX = flex -YACC = bison -MAKE = make -INSTALL ?= install - -CFLAGS += -Wall -O2 -CFLAGS += -D__EXPORTED_HEADERS__ -I$(srctree)/tools/include/uapi \ - -I$(srctree)/tools/include -CFLAGS += $(EXTRA_CFLAGS) - -# This will work when bpf is built in tools env. where srctree -# isn't set and when invoked from selftests build, where srctree -# is set to ".". building_out_of_srctree is undefined for in srctree -# builds -ifeq ($(srctree),) -update_srctree := 1 -endif -ifndef building_out_of_srctree -update_srctree := 1 -endif -ifeq ($(update_srctree),1) -srctree := $(patsubst %/,%,$(dir $(CURDIR))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -endif - -FEATURE_USER = .bpf -FEATURE_TESTS = libbfd disassembler-four-args disassembler-init-styled -FEATURE_DISPLAY = libbfd - -check_feat := 1 -NON_CHECK_FEAT_TARGETS := clean bpftool_clean resolve_btfids_clean -ifdef MAKECMDGOALS -ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),) - check_feat := 0 -endif -endif - -ifeq ($(check_feat),1) -ifeq ($(FEATURES_DUMP),) -include $(srctree)/tools/build/Makefile.feature -else -include $(FEATURES_DUMP) -endif -endif - -ifeq ($(feature-disassembler-four-args), 1) -CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE -endif -ifeq ($(feature-disassembler-init-styled), 1) -CFLAGS += -DDISASM_INIT_STYLED -endif - -$(OUTPUT)%.yacc.c: $(srctree)/tools/bpf/%.y - $(QUIET_BISON)$(YACC) -o $@ -d $< - -$(OUTPUT)%.lex.c: $(srctree)/tools/bpf/%.l - $(QUIET_FLEX)$(LEX) -o $@ $< - -$(OUTPUT)%.o: $(srctree)/tools/bpf/%.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< - -$(OUTPUT)%.yacc.o: $(OUTPUT)%.yacc.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< -$(OUTPUT)%.lex.o: $(OUTPUT)%.lex.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< - -PROGS = $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg $(OUTPUT)bpf_asm - -all: $(PROGS) bpftool - -$(OUTPUT)bpf_jit_disasm: CFLAGS += -DPACKAGE='bpf_jit_disasm' -$(OUTPUT)bpf_jit_disasm: $(OUTPUT)bpf_jit_disasm.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ -lopcodes -lbfd -ldl - -$(OUTPUT)bpf_dbg: $(OUTPUT)bpf_dbg.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ -lreadline - -$(OUTPUT)bpf_asm: $(OUTPUT)bpf_asm.o $(OUTPUT)bpf_exp.yacc.o $(OUTPUT)bpf_exp.lex.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ - -$(OUTPUT)bpf_exp.lex.c: $(OUTPUT)bpf_exp.yacc.c -$(OUTPUT)bpf_exp.yacc.o: $(OUTPUT)bpf_exp.yacc.c -$(OUTPUT)bpf_exp.lex.o: $(OUTPUT)bpf_exp.lex.c +all: bpftool clean: bpftool_clean resolve_btfids_clean - $(call QUIET_CLEAN, bpf-progs) - $(Q)$(RM) -r -- $(OUTPUT)*.o $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg \ - $(OUTPUT)bpf_asm $(OUTPUT)bpf_exp.yacc.* $(OUTPUT)bpf_exp.lex.* - $(call QUIET_CLEAN, core-gen) - $(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpf - $(Q)$(RM) -r -- $(OUTPUT)feature -install: $(PROGS) bpftool_install - $(call QUIET_INSTALL, bpf_jit_disasm) - $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/bin - $(Q)$(INSTALL) $(OUTPUT)bpf_jit_disasm $(DESTDIR)$(prefix)/bin/bpf_jit_disasm - $(call QUIET_INSTALL, bpf_dbg) - $(Q)$(INSTALL) $(OUTPUT)bpf_dbg $(DESTDIR)$(prefix)/bin/bpf_dbg - $(call QUIET_INSTALL, bpf_asm) - $(Q)$(INSTALL) $(OUTPUT)bpf_asm $(DESTDIR)$(prefix)/bin/bpf_asm +install: bpftool_install bpftool: $(call descend,bpftool)
diff --git a/tools/bpf/bpf_asm.c b/tools/bpf/bpf_asm.c deleted file mode 100644 index 0063c3c..0000000 --- a/tools/bpf/bpf_asm.c +++ /dev/null
@@ -1,52 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF assembler - * - * Instead of libpcap high-level filter expressions, it can be quite - * useful to define filters in low-level BPF assembler (that is kept - * close to Steven McCanne and Van Jacobson's original BPF paper). - * In particular for BPF JIT implementors, JIT security auditors, or - * just for defining BPF expressions that contain extensions which are - * not supported by compilers. - * - * How to get into it: - * - * 1) read Documentation/networking/filter.rst - * 2) Run `bpf_asm [-c] <filter-prog file>` to translate into binary - * blob that is loadable with xt_bpf, cls_bpf et al. Note: -c will - * pretty print a C-like construct. - * - * Copyright 2013 Daniel Borkmann <borkmann@redhat.com> - */ - -#include <stdbool.h> -#include <stdio.h> -#include <string.h> - -extern void bpf_asm_compile(FILE *fp, bool cstyle); - -int main(int argc, char **argv) -{ - FILE *fp = stdin; - bool cstyle = false; - int i; - - for (i = 1; i < argc; i++) { - if (!strncmp("-c", argv[i], 2)) { - cstyle = true; - continue; - } - - fp = fopen(argv[i], "r"); - if (!fp) { - fp = stdin; - continue; - } - - break; - } - - bpf_asm_compile(fp, cstyle); - - return 0; -}
diff --git a/tools/bpf/bpf_dbg.c b/tools/bpf/bpf_dbg.c deleted file mode 100644 index 00e560a..0000000 --- a/tools/bpf/bpf_dbg.c +++ /dev/null
@@ -1,1398 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF debugger - * - * Minimal BPF debugger that mimics the kernel's engine (w/o extensions) - * and allows for single stepping through selected packets from a pcap - * with a provided user filter in order to facilitate verification of a - * BPF program. Besides others, this is useful to verify BPF programs - * before attaching to a live system, and can be used in socket filters, - * cls_bpf, xt_bpf, team driver and e.g. PTP code; in particular when a - * single more complex BPF program is being used. Reasons for a more - * complex BPF program are likely primarily to optimize execution time - * for making a verdict when multiple simple BPF programs are combined - * into one in order to prevent parsing same headers multiple times. - * - * More on how to debug BPF opcodes see Documentation/networking/filter.rst - * which is the main document on BPF. Mini howto for getting started: - * - * 1) `./bpf_dbg` to enter the shell (shell cmds denoted with '>'): - * 2) > load bpf 6,40 0 0 12,21 0 3 20... (output from `bpf_asm` or - * `tcpdump -iem1 -ddd port 22 | tr '\n' ','` to load as filter) - * 3) > load pcap foo.pcap - * 4) > run <n>/disassemble/dump/quit (self-explanatory) - * 5) > breakpoint 2 (sets bp at loaded BPF insns 2, do `run` then; - * multiple bps can be set, of course, a call to `breakpoint` - * w/o args shows currently loaded bps, `breakpoint reset` for - * resetting all breakpoints) - * 6) > select 3 (`run` etc will start from the 3rd packet in the pcap) - * 7) > step [-<n>, +<n>] (performs single stepping through the BPF) - * - * Copyright 2013 Daniel Borkmann <borkmann@redhat.com> - */ - -#include <stdio.h> -#include <unistd.h> -#include <stdlib.h> -#include <ctype.h> -#include <stdbool.h> -#include <stdarg.h> -#include <setjmp.h> -#include <linux/filter.h> -#include <linux/if_packet.h> -#include <readline/readline.h> -#include <readline/history.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/stat.h> -#include <sys/mman.h> -#include <fcntl.h> -#include <errno.h> -#include <signal.h> -#include <arpa/inet.h> -#include <net/ethernet.h> - -#define TCPDUMP_MAGIC 0xa1b2c3d4 - -#define BPF_LDX_B (BPF_LDX | BPF_B) -#define BPF_LDX_W (BPF_LDX | BPF_W) -#define BPF_JMP_JA (BPF_JMP | BPF_JA) -#define BPF_JMP_JEQ (BPF_JMP | BPF_JEQ) -#define BPF_JMP_JGT (BPF_JMP | BPF_JGT) -#define BPF_JMP_JGE (BPF_JMP | BPF_JGE) -#define BPF_JMP_JSET (BPF_JMP | BPF_JSET) -#define BPF_ALU_ADD (BPF_ALU | BPF_ADD) -#define BPF_ALU_SUB (BPF_ALU | BPF_SUB) -#define BPF_ALU_MUL (BPF_ALU | BPF_MUL) -#define BPF_ALU_DIV (BPF_ALU | BPF_DIV) -#define BPF_ALU_MOD (BPF_ALU | BPF_MOD) -#define BPF_ALU_NEG (BPF_ALU | BPF_NEG) -#define BPF_ALU_AND (BPF_ALU | BPF_AND) -#define BPF_ALU_OR (BPF_ALU | BPF_OR) -#define BPF_ALU_XOR (BPF_ALU | BPF_XOR) -#define BPF_ALU_LSH (BPF_ALU | BPF_LSH) -#define BPF_ALU_RSH (BPF_ALU | BPF_RSH) -#define BPF_MISC_TAX (BPF_MISC | BPF_TAX) -#define BPF_MISC_TXA (BPF_MISC | BPF_TXA) -#define BPF_LD_B (BPF_LD | BPF_B) -#define BPF_LD_H (BPF_LD | BPF_H) -#define BPF_LD_W (BPF_LD | BPF_W) - -#ifndef array_size -# define array_size(x) (sizeof(x) / sizeof((x)[0])) -#endif - -#ifndef __check_format_printf -# define __check_format_printf(pos_fmtstr, pos_fmtargs) \ - __attribute__ ((format (printf, (pos_fmtstr), (pos_fmtargs)))) -#endif - -enum { - CMD_OK, - CMD_ERR, - CMD_EX, -}; - -struct shell_cmd { - const char *name; - int (*func)(char *args); -}; - -struct pcap_filehdr { - uint32_t magic; - uint16_t version_major; - uint16_t version_minor; - int32_t thiszone; - uint32_t sigfigs; - uint32_t snaplen; - uint32_t linktype; -}; - -struct pcap_timeval { - int32_t tv_sec; - int32_t tv_usec; -}; - -struct pcap_pkthdr { - struct pcap_timeval ts; - uint32_t caplen; - uint32_t len; -}; - -struct bpf_regs { - uint32_t A; - uint32_t X; - uint32_t M[BPF_MEMWORDS]; - uint32_t R; - bool Rs; - uint16_t Pc; -}; - -static struct sock_filter bpf_image[BPF_MAXINSNS + 1]; -static unsigned int bpf_prog_len; - -static int bpf_breakpoints[64]; -static struct bpf_regs bpf_regs[BPF_MAXINSNS + 1]; -static struct bpf_regs bpf_curr; -static unsigned int bpf_regs_len; - -static int pcap_fd = -1; -static unsigned int pcap_packet; -static size_t pcap_map_size; -static char *pcap_ptr_va_start, *pcap_ptr_va_curr; - -static const char * const op_table[] = { - [BPF_ST] = "st", - [BPF_STX] = "stx", - [BPF_LD_B] = "ldb", - [BPF_LD_H] = "ldh", - [BPF_LD_W] = "ld", - [BPF_LDX] = "ldx", - [BPF_LDX_B] = "ldxb", - [BPF_JMP_JA] = "ja", - [BPF_JMP_JEQ] = "jeq", - [BPF_JMP_JGT] = "jgt", - [BPF_JMP_JGE] = "jge", - [BPF_JMP_JSET] = "jset", - [BPF_ALU_ADD] = "add", - [BPF_ALU_SUB] = "sub", - [BPF_ALU_MUL] = "mul", - [BPF_ALU_DIV] = "div", - [BPF_ALU_MOD] = "mod", - [BPF_ALU_NEG] = "neg", - [BPF_ALU_AND] = "and", - [BPF_ALU_OR] = "or", - [BPF_ALU_XOR] = "xor", - [BPF_ALU_LSH] = "lsh", - [BPF_ALU_RSH] = "rsh", - [BPF_MISC_TAX] = "tax", - [BPF_MISC_TXA] = "txa", - [BPF_RET] = "ret", -}; - -static __check_format_printf(1, 2) int rl_printf(const char *fmt, ...) -{ - int ret; - va_list vl; - - va_start(vl, fmt); - ret = vfprintf(rl_outstream, fmt, vl); - va_end(vl); - - return ret; -} - -static int matches(const char *cmd, const char *pattern) -{ - int len = strlen(cmd); - - if (len > strlen(pattern)) - return -1; - - return memcmp(pattern, cmd, len); -} - -static void hex_dump(const uint8_t *buf, size_t len) -{ - int i; - - rl_printf("%3u: ", 0); - for (i = 0; i < len; i++) { - if (i && !(i % 16)) - rl_printf("\n%3u: ", i); - rl_printf("%02x ", buf[i]); - } - rl_printf("\n"); -} - -static bool bpf_prog_loaded(void) -{ - if (bpf_prog_len == 0) - rl_printf("no bpf program loaded!\n"); - - return bpf_prog_len > 0; -} - -static void bpf_disasm(const struct sock_filter f, unsigned int i) -{ - const char *op, *fmt; - int val = f.k; - char buf[256]; - - switch (f.code) { - case BPF_RET | BPF_K: - op = op_table[BPF_RET]; - fmt = "#%#x"; - break; - case BPF_RET | BPF_A: - op = op_table[BPF_RET]; - fmt = "a"; - break; - case BPF_RET | BPF_X: - op = op_table[BPF_RET]; - fmt = "x"; - break; - case BPF_MISC_TAX: - op = op_table[BPF_MISC_TAX]; - fmt = ""; - break; - case BPF_MISC_TXA: - op = op_table[BPF_MISC_TXA]; - fmt = ""; - break; - case BPF_ST: - op = op_table[BPF_ST]; - fmt = "M[%d]"; - break; - case BPF_STX: - op = op_table[BPF_STX]; - fmt = "M[%d]"; - break; - case BPF_LD_W | BPF_ABS: - op = op_table[BPF_LD_W]; - fmt = "[%d]"; - break; - case BPF_LD_H | BPF_ABS: - op = op_table[BPF_LD_H]; - fmt = "[%d]"; - break; - case BPF_LD_B | BPF_ABS: - op = op_table[BPF_LD_B]; - fmt = "[%d]"; - break; - case BPF_LD_W | BPF_LEN: - op = op_table[BPF_LD_W]; - fmt = "#len"; - break; - case BPF_LD_W | BPF_IND: - op = op_table[BPF_LD_W]; - fmt = "[x+%d]"; - break; - case BPF_LD_H | BPF_IND: - op = op_table[BPF_LD_H]; - fmt = "[x+%d]"; - break; - case BPF_LD_B | BPF_IND: - op = op_table[BPF_LD_B]; - fmt = "[x+%d]"; - break; - case BPF_LD | BPF_IMM: - op = op_table[BPF_LD_W]; - fmt = "#%#x"; - break; - case BPF_LDX | BPF_IMM: - op = op_table[BPF_LDX]; - fmt = "#%#x"; - break; - case BPF_LDX_B | BPF_MSH: - op = op_table[BPF_LDX_B]; - fmt = "4*([%d]&0xf)"; - break; - case BPF_LD | BPF_MEM: - op = op_table[BPF_LD_W]; - fmt = "M[%d]"; - break; - case BPF_LDX | BPF_MEM: - op = op_table[BPF_LDX]; - fmt = "M[%d]"; - break; - case BPF_JMP_JA: - op = op_table[BPF_JMP_JA]; - fmt = "%d"; - val = i + 1 + f.k; - break; - case BPF_JMP_JGT | BPF_X: - op = op_table[BPF_JMP_JGT]; - fmt = "x"; - break; - case BPF_JMP_JGT | BPF_K: - op = op_table[BPF_JMP_JGT]; - fmt = "#%#x"; - break; - case BPF_JMP_JGE | BPF_X: - op = op_table[BPF_JMP_JGE]; - fmt = "x"; - break; - case BPF_JMP_JGE | BPF_K: - op = op_table[BPF_JMP_JGE]; - fmt = "#%#x"; - break; - case BPF_JMP_JEQ | BPF_X: - op = op_table[BPF_JMP_JEQ]; - fmt = "x"; - break; - case BPF_JMP_JEQ | BPF_K: - op = op_table[BPF_JMP_JEQ]; - fmt = "#%#x"; - break; - case BPF_JMP_JSET | BPF_X: - op = op_table[BPF_JMP_JSET]; - fmt = "x"; - break; - case BPF_JMP_JSET | BPF_K: - op = op_table[BPF_JMP_JSET]; - fmt = "#%#x"; - break; - case BPF_ALU_NEG: - op = op_table[BPF_ALU_NEG]; - fmt = ""; - break; - case BPF_ALU_LSH | BPF_X: - op = op_table[BPF_ALU_LSH]; - fmt = "x"; - break; - case BPF_ALU_LSH | BPF_K: - op = op_table[BPF_ALU_LSH]; - fmt = "#%d"; - break; - case BPF_ALU_RSH | BPF_X: - op = op_table[BPF_ALU_RSH]; - fmt = "x"; - break; - case BPF_ALU_RSH | BPF_K: - op = op_table[BPF_ALU_RSH]; - fmt = "#%d"; - break; - case BPF_ALU_ADD | BPF_X: - op = op_table[BPF_ALU_ADD]; - fmt = "x"; - break; - case BPF_ALU_ADD | BPF_K: - op = op_table[BPF_ALU_ADD]; - fmt = "#%d"; - break; - case BPF_ALU_SUB | BPF_X: - op = op_table[BPF_ALU_SUB]; - fmt = "x"; - break; - case BPF_ALU_SUB | BPF_K: - op = op_table[BPF_ALU_SUB]; - fmt = "#%d"; - break; - case BPF_ALU_MUL | BPF_X: - op = op_table[BPF_ALU_MUL]; - fmt = "x"; - break; - case BPF_ALU_MUL | BPF_K: - op = op_table[BPF_ALU_MUL]; - fmt = "#%d"; - break; - case BPF_ALU_DIV | BPF_X: - op = op_table[BPF_ALU_DIV]; - fmt = "x"; - break; - case BPF_ALU_DIV | BPF_K: - op = op_table[BPF_ALU_DIV]; - fmt = "#%d"; - break; - case BPF_ALU_MOD | BPF_X: - op = op_table[BPF_ALU_MOD]; - fmt = "x"; - break; - case BPF_ALU_MOD | BPF_K: - op = op_table[BPF_ALU_MOD]; - fmt = "#%d"; - break; - case BPF_ALU_AND | BPF_X: - op = op_table[BPF_ALU_AND]; - fmt = "x"; - break; - case BPF_ALU_AND | BPF_K: - op = op_table[BPF_ALU_AND]; - fmt = "#%#x"; - break; - case BPF_ALU_OR | BPF_X: - op = op_table[BPF_ALU_OR]; - fmt = "x"; - break; - case BPF_ALU_OR | BPF_K: - op = op_table[BPF_ALU_OR]; - fmt = "#%#x"; - break; - case BPF_ALU_XOR | BPF_X: - op = op_table[BPF_ALU_XOR]; - fmt = "x"; - break; - case BPF_ALU_XOR | BPF_K: - op = op_table[BPF_ALU_XOR]; - fmt = "#%#x"; - break; - default: - op = "nosup"; - fmt = "%#x"; - val = f.code; - break; - } - - memset(buf, 0, sizeof(buf)); - snprintf(buf, sizeof(buf), fmt, val); - buf[sizeof(buf) - 1] = 0; - - if ((BPF_CLASS(f.code) == BPF_JMP && BPF_OP(f.code) != BPF_JA)) - rl_printf("l%d:\t%s %s, l%d, l%d\n", i, op, buf, - i + 1 + f.jt, i + 1 + f.jf); - else - rl_printf("l%d:\t%s %s\n", i, op, buf); -} - -static void bpf_dump_curr(struct bpf_regs *r, struct sock_filter *f) -{ - int i, m = 0; - - rl_printf("pc: [%u]\n", r->Pc); - rl_printf("code: [%u] jt[%u] jf[%u] k[%u]\n", - f->code, f->jt, f->jf, f->k); - rl_printf("curr: "); - bpf_disasm(*f, r->Pc); - - if (f->jt || f->jf) { - rl_printf("jt: "); - bpf_disasm(*(f + f->jt + 1), r->Pc + f->jt + 1); - rl_printf("jf: "); - bpf_disasm(*(f + f->jf + 1), r->Pc + f->jf + 1); - } - - rl_printf("A: [%#08x][%u]\n", r->A, r->A); - rl_printf("X: [%#08x][%u]\n", r->X, r->X); - if (r->Rs) - rl_printf("ret: [%#08x][%u]!\n", r->R, r->R); - - for (i = 0; i < BPF_MEMWORDS; i++) { - if (r->M[i]) { - m++; - rl_printf("M[%d]: [%#08x][%u]\n", i, r->M[i], r->M[i]); - } - } - if (m == 0) - rl_printf("M[0,%d]: [%#08x][%u]\n", BPF_MEMWORDS - 1, 0, 0); -} - -static void bpf_dump_pkt(uint8_t *pkt, uint32_t pkt_caplen, uint32_t pkt_len) -{ - if (pkt_caplen != pkt_len) - rl_printf("cap: %u, len: %u\n", pkt_caplen, pkt_len); - else - rl_printf("len: %u\n", pkt_len); - - hex_dump(pkt, pkt_caplen); -} - -static void bpf_disasm_all(const struct sock_filter *f, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - bpf_disasm(f[i], i); -} - -static void bpf_dump_all(const struct sock_filter *f, unsigned int len) -{ - unsigned int i; - - rl_printf("/* { op, jt, jf, k }, */\n"); - for (i = 0; i < len; i++) - rl_printf("{ %#04x, %2u, %2u, %#010x },\n", - f[i].code, f[i].jt, f[i].jf, f[i].k); -} - -static bool bpf_runnable(struct sock_filter *f, unsigned int len) -{ - int sock, ret, i; - struct sock_fprog bpf = { - .filter = f, - .len = len, - }; - - sock = socket(AF_INET, SOCK_DGRAM, 0); - if (sock < 0) { - rl_printf("cannot open socket!\n"); - return false; - } - ret = setsockopt(sock, SOL_SOCKET, SO_ATTACH_FILTER, &bpf, sizeof(bpf)); - close(sock); - if (ret < 0) { - rl_printf("program not allowed to run by kernel!\n"); - return false; - } - for (i = 0; i < len; i++) { - if (BPF_CLASS(f[i].code) == BPF_LD && - f[i].k > SKF_AD_OFF) { - rl_printf("extensions currently not supported!\n"); - return false; - } - } - - return true; -} - -static void bpf_reset_breakpoints(void) -{ - int i; - - for (i = 0; i < array_size(bpf_breakpoints); i++) - bpf_breakpoints[i] = -1; -} - -static void bpf_set_breakpoints(unsigned int where) -{ - int i; - bool set = false; - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] == (int) where) { - rl_printf("breakpoint already set!\n"); - set = true; - break; - } - - if (bpf_breakpoints[i] == -1 && set == false) { - bpf_breakpoints[i] = where; - set = true; - } - } - - if (!set) - rl_printf("too many breakpoints set, reset first!\n"); -} - -static void bpf_dump_breakpoints(void) -{ - int i; - - rl_printf("breakpoints: "); - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] < 0) - continue; - rl_printf("%d ", bpf_breakpoints[i]); - } - - rl_printf("\n"); -} - -static void bpf_reset(void) -{ - bpf_regs_len = 0; - - memset(bpf_regs, 0, sizeof(bpf_regs)); - memset(&bpf_curr, 0, sizeof(bpf_curr)); -} - -static void bpf_safe_regs(void) -{ - memcpy(&bpf_regs[bpf_regs_len++], &bpf_curr, sizeof(bpf_curr)); -} - -static bool bpf_restore_regs(int off) -{ - unsigned int index = bpf_regs_len - 1 + off; - - if (index == 0) { - bpf_reset(); - return true; - } else if (index < bpf_regs_len) { - memcpy(&bpf_curr, &bpf_regs[index], sizeof(bpf_curr)); - bpf_regs_len = index; - return true; - } else { - rl_printf("reached bottom of register history stack!\n"); - return false; - } -} - -static uint32_t extract_u32(uint8_t *pkt, uint32_t off) -{ - uint32_t r; - - memcpy(&r, &pkt[off], sizeof(r)); - - return ntohl(r); -} - -static uint16_t extract_u16(uint8_t *pkt, uint32_t off) -{ - uint16_t r; - - memcpy(&r, &pkt[off], sizeof(r)); - - return ntohs(r); -} - -static uint8_t extract_u8(uint8_t *pkt, uint32_t off) -{ - return pkt[off]; -} - -static void set_return(struct bpf_regs *r) -{ - r->R = 0; - r->Rs = true; -} - -static void bpf_single_step(struct bpf_regs *r, struct sock_filter *f, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len) -{ - uint32_t K = f->k; - int d; - - switch (f->code) { - case BPF_RET | BPF_K: - r->R = K; - r->Rs = true; - break; - case BPF_RET | BPF_A: - r->R = r->A; - r->Rs = true; - break; - case BPF_RET | BPF_X: - r->R = r->X; - r->Rs = true; - break; - case BPF_MISC_TAX: - r->X = r->A; - break; - case BPF_MISC_TXA: - r->A = r->X; - break; - case BPF_ST: - r->M[K] = r->A; - break; - case BPF_STX: - r->M[K] = r->X; - break; - case BPF_LD_W | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint32_t)) - r->A = extract_u32(pkt, K); - else - set_return(r); - break; - case BPF_LD_H | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint16_t)) - r->A = extract_u16(pkt, K); - else - set_return(r); - break; - case BPF_LD_B | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint8_t)) - r->A = extract_u8(pkt, K); - else - set_return(r); - break; - case BPF_LD_W | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint32_t)) - r->A = extract_u32(pkt, r->X + K); - break; - case BPF_LD_H | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint16_t)) - r->A = extract_u16(pkt, r->X + K); - else - set_return(r); - break; - case BPF_LD_B | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint8_t)) - r->A = extract_u8(pkt, r->X + K); - else - set_return(r); - break; - case BPF_LDX_B | BPF_MSH: - d = pkt_caplen - K; - if (d >= sizeof(uint8_t)) { - r->X = extract_u8(pkt, K); - r->X = (r->X & 0xf) << 2; - } else - set_return(r); - break; - case BPF_LD_W | BPF_LEN: - r->A = pkt_len; - break; - case BPF_LDX_W | BPF_LEN: - r->A = pkt_len; - break; - case BPF_LD | BPF_IMM: - r->A = K; - break; - case BPF_LDX | BPF_IMM: - r->X = K; - break; - case BPF_LD | BPF_MEM: - r->A = r->M[K]; - break; - case BPF_LDX | BPF_MEM: - r->X = r->M[K]; - break; - case BPF_JMP_JA: - r->Pc += K; - break; - case BPF_JMP_JGT | BPF_X: - r->Pc += r->A > r->X ? f->jt : f->jf; - break; - case BPF_JMP_JGT | BPF_K: - r->Pc += r->A > K ? f->jt : f->jf; - break; - case BPF_JMP_JGE | BPF_X: - r->Pc += r->A >= r->X ? f->jt : f->jf; - break; - case BPF_JMP_JGE | BPF_K: - r->Pc += r->A >= K ? f->jt : f->jf; - break; - case BPF_JMP_JEQ | BPF_X: - r->Pc += r->A == r->X ? f->jt : f->jf; - break; - case BPF_JMP_JEQ | BPF_K: - r->Pc += r->A == K ? f->jt : f->jf; - break; - case BPF_JMP_JSET | BPF_X: - r->Pc += r->A & r->X ? f->jt : f->jf; - break; - case BPF_JMP_JSET | BPF_K: - r->Pc += r->A & K ? f->jt : f->jf; - break; - case BPF_ALU_NEG: - r->A = -r->A; - break; - case BPF_ALU_LSH | BPF_X: - r->A <<= r->X; - break; - case BPF_ALU_LSH | BPF_K: - r->A <<= K; - break; - case BPF_ALU_RSH | BPF_X: - r->A >>= r->X; - break; - case BPF_ALU_RSH | BPF_K: - r->A >>= K; - break; - case BPF_ALU_ADD | BPF_X: - r->A += r->X; - break; - case BPF_ALU_ADD | BPF_K: - r->A += K; - break; - case BPF_ALU_SUB | BPF_X: - r->A -= r->X; - break; - case BPF_ALU_SUB | BPF_K: - r->A -= K; - break; - case BPF_ALU_MUL | BPF_X: - r->A *= r->X; - break; - case BPF_ALU_MUL | BPF_K: - r->A *= K; - break; - case BPF_ALU_DIV | BPF_X: - case BPF_ALU_MOD | BPF_X: - if (r->X == 0) { - set_return(r); - break; - } - goto do_div; - case BPF_ALU_DIV | BPF_K: - case BPF_ALU_MOD | BPF_K: - if (K == 0) { - set_return(r); - break; - } -do_div: - switch (f->code) { - case BPF_ALU_DIV | BPF_X: - r->A /= r->X; - break; - case BPF_ALU_DIV | BPF_K: - r->A /= K; - break; - case BPF_ALU_MOD | BPF_X: - r->A %= r->X; - break; - case BPF_ALU_MOD | BPF_K: - r->A %= K; - break; - } - break; - case BPF_ALU_AND | BPF_X: - r->A &= r->X; - break; - case BPF_ALU_AND | BPF_K: - r->A &= K; - break; - case BPF_ALU_OR | BPF_X: - r->A |= r->X; - break; - case BPF_ALU_OR | BPF_K: - r->A |= K; - break; - case BPF_ALU_XOR | BPF_X: - r->A ^= r->X; - break; - case BPF_ALU_XOR | BPF_K: - r->A ^= K; - break; - } -} - -static bool bpf_pc_has_breakpoint(uint16_t pc) -{ - int i; - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] < 0) - continue; - if (bpf_breakpoints[i] == pc) - return true; - } - - return false; -} - -static bool bpf_handle_breakpoint(struct bpf_regs *r, struct sock_filter *f, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len) -{ - rl_printf("-- register dump --\n"); - bpf_dump_curr(r, &f[r->Pc]); - rl_printf("-- packet dump --\n"); - bpf_dump_pkt(pkt, pkt_caplen, pkt_len); - rl_printf("(breakpoint)\n"); - return true; -} - -static int bpf_run_all(struct sock_filter *f, uint16_t bpf_len, uint8_t *pkt, - uint32_t pkt_caplen, uint32_t pkt_len) -{ - bool stop = false; - - while (bpf_curr.Rs == false && stop == false) { - bpf_safe_regs(); - - if (bpf_pc_has_breakpoint(bpf_curr.Pc)) - stop = bpf_handle_breakpoint(&bpf_curr, f, pkt, - pkt_caplen, pkt_len); - - bpf_single_step(&bpf_curr, &f[bpf_curr.Pc], pkt, pkt_caplen, - pkt_len); - bpf_curr.Pc++; - } - - return stop ? -1 : bpf_curr.R; -} - -static int bpf_run_stepping(struct sock_filter *f, uint16_t bpf_len, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len, int next) -{ - bool stop = false; - int i = 1; - - while (!bpf_curr.Rs && !stop) { - bpf_safe_regs(); - - if (i++ == next) - stop = bpf_handle_breakpoint(&bpf_curr, f, pkt, - pkt_caplen, pkt_len); - - bpf_single_step(&bpf_curr, &f[bpf_curr.Pc], pkt, pkt_caplen, - pkt_len); - bpf_curr.Pc++; - } - - return stop ? -1 : bpf_curr.R; -} - -static bool pcap_loaded(void) -{ - if (pcap_fd < 0) - rl_printf("no pcap file loaded!\n"); - - return pcap_fd >= 0; -} - -static struct pcap_pkthdr *pcap_curr_pkt(void) -{ - return (void *) pcap_ptr_va_curr; -} - -static bool pcap_next_pkt(void) -{ - struct pcap_pkthdr *hdr = pcap_curr_pkt(); - - if (pcap_ptr_va_curr + sizeof(*hdr) - - pcap_ptr_va_start >= pcap_map_size) - return false; - if (hdr->caplen == 0 || hdr->len == 0 || hdr->caplen > hdr->len) - return false; - if (pcap_ptr_va_curr + sizeof(*hdr) + hdr->caplen - - pcap_ptr_va_start >= pcap_map_size) - return false; - - pcap_ptr_va_curr += (sizeof(*hdr) + hdr->caplen); - return true; -} - -static void pcap_reset_pkt(void) -{ - pcap_ptr_va_curr = pcap_ptr_va_start + sizeof(struct pcap_filehdr); -} - -static int try_load_pcap(const char *file) -{ - struct pcap_filehdr *hdr; - struct stat sb; - int ret; - - pcap_fd = open(file, O_RDONLY); - if (pcap_fd < 0) { - rl_printf("cannot open pcap [%s]!\n", strerror(errno)); - return CMD_ERR; - } - - ret = fstat(pcap_fd, &sb); - if (ret < 0) { - rl_printf("cannot fstat pcap file!\n"); - return CMD_ERR; - } - - if (!S_ISREG(sb.st_mode)) { - rl_printf("not a regular pcap file, duh!\n"); - return CMD_ERR; - } - - pcap_map_size = sb.st_size; - if (pcap_map_size <= sizeof(struct pcap_filehdr)) { - rl_printf("pcap file too small!\n"); - return CMD_ERR; - } - - pcap_ptr_va_start = mmap(NULL, pcap_map_size, PROT_READ, - MAP_SHARED | MAP_LOCKED, pcap_fd, 0); - if (pcap_ptr_va_start == MAP_FAILED) { - rl_printf("mmap of file failed!"); - return CMD_ERR; - } - - hdr = (void *) pcap_ptr_va_start; - if (hdr->magic != TCPDUMP_MAGIC) { - rl_printf("wrong pcap magic!\n"); - return CMD_ERR; - } - - pcap_reset_pkt(); - - return CMD_OK; - -} - -static void try_close_pcap(void) -{ - if (pcap_fd >= 0) { - munmap(pcap_ptr_va_start, pcap_map_size); - close(pcap_fd); - - pcap_ptr_va_start = pcap_ptr_va_curr = NULL; - pcap_map_size = 0; - pcap_packet = 0; - pcap_fd = -1; - } -} - -static int cmd_load_bpf(char *bpf_string) -{ - char sp, *token, separator = ','; - unsigned short bpf_len, i = 0; - struct sock_filter tmp; - - bpf_prog_len = 0; - memset(bpf_image, 0, sizeof(bpf_image)); - - if (sscanf(bpf_string, "%hu%c", &bpf_len, &sp) != 2 || - sp != separator || bpf_len > BPF_MAXINSNS || bpf_len == 0) { - rl_printf("syntax error in head length encoding!\n"); - return CMD_ERR; - } - - token = bpf_string; - while ((token = strchr(token, separator)) && (++token)[0]) { - if (i >= bpf_len) { - rl_printf("program exceeds encoded length!\n"); - return CMD_ERR; - } - - if (sscanf(token, "%hu %hhu %hhu %u,", - &tmp.code, &tmp.jt, &tmp.jf, &tmp.k) != 4) { - rl_printf("syntax error at instruction %d!\n", i); - return CMD_ERR; - } - - bpf_image[i].code = tmp.code; - bpf_image[i].jt = tmp.jt; - bpf_image[i].jf = tmp.jf; - bpf_image[i].k = tmp.k; - - i++; - } - - if (i != bpf_len) { - rl_printf("syntax error exceeding encoded length!\n"); - return CMD_ERR; - } else - bpf_prog_len = bpf_len; - if (!bpf_runnable(bpf_image, bpf_prog_len)) - bpf_prog_len = 0; - - return CMD_OK; -} - -static int cmd_load_pcap(char *file) -{ - char *file_trim, *tmp; - - file_trim = strtok_r(file, " ", &tmp); - if (file_trim == NULL) - return CMD_ERR; - - try_close_pcap(); - - return try_load_pcap(file_trim); -} - -static int cmd_load(char *arg) -{ - char *subcmd, *cont = NULL, *tmp = strdup(arg); - int ret = CMD_OK; - - subcmd = strtok_r(tmp, " ", &cont); - if (subcmd == NULL) - goto out; - if (matches(subcmd, "bpf") == 0) { - bpf_reset(); - bpf_reset_breakpoints(); - - if (!cont) - ret = CMD_ERR; - else - ret = cmd_load_bpf(cont); - } else if (matches(subcmd, "pcap") == 0) { - ret = cmd_load_pcap(cont); - } else { -out: - rl_printf("bpf <code>: load bpf code\n"); - rl_printf("pcap <file>: load pcap file\n"); - ret = CMD_ERR; - } - - free(tmp); - return ret; -} - -static int cmd_step(char *num) -{ - struct pcap_pkthdr *hdr; - int steps, ret; - - if (!bpf_prog_loaded() || !pcap_loaded()) - return CMD_ERR; - - steps = strtol(num, NULL, 10); - if (steps == 0 || strlen(num) == 0) - steps = 1; - if (steps < 0) { - if (!bpf_restore_regs(steps)) - return CMD_ERR; - steps = 1; - } - - hdr = pcap_curr_pkt(); - ret = bpf_run_stepping(bpf_image, bpf_prog_len, - (uint8_t *) hdr + sizeof(*hdr), - hdr->caplen, hdr->len, steps); - if (ret >= 0 || bpf_curr.Rs) { - bpf_reset(); - if (!pcap_next_pkt()) { - rl_printf("(going back to first packet)\n"); - pcap_reset_pkt(); - } else { - rl_printf("(next packet)\n"); - } - } - - return CMD_OK; -} - -static int cmd_select(char *num) -{ - unsigned int which, i; - bool have_next = true; - - if (!pcap_loaded() || strlen(num) == 0) - return CMD_ERR; - - which = strtoul(num, NULL, 10); - if (which == 0) { - rl_printf("packet count starts with 1, clamping!\n"); - which = 1; - } - - pcap_reset_pkt(); - bpf_reset(); - - for (i = 0; i < which && (have_next = pcap_next_pkt()); i++) - /* noop */; - if (!have_next || pcap_curr_pkt() == NULL) { - rl_printf("no packet #%u available!\n", which); - pcap_reset_pkt(); - return CMD_ERR; - } - - return CMD_OK; -} - -static int cmd_breakpoint(char *subcmd) -{ - if (!bpf_prog_loaded()) - return CMD_ERR; - if (strlen(subcmd) == 0) - bpf_dump_breakpoints(); - else if (matches(subcmd, "reset") == 0) - bpf_reset_breakpoints(); - else { - unsigned int where = strtoul(subcmd, NULL, 10); - - if (where < bpf_prog_len) { - bpf_set_breakpoints(where); - rl_printf("breakpoint at: "); - bpf_disasm(bpf_image[where], where); - } - } - - return CMD_OK; -} - -static int cmd_run(char *num) -{ - static uint32_t pass, fail; - bool has_limit = true; - int pkts = 0, i = 0; - - if (!bpf_prog_loaded() || !pcap_loaded()) - return CMD_ERR; - - pkts = strtol(num, NULL, 10); - if (pkts == 0 || strlen(num) == 0) - has_limit = false; - - do { - struct pcap_pkthdr *hdr = pcap_curr_pkt(); - int ret = bpf_run_all(bpf_image, bpf_prog_len, - (uint8_t *) hdr + sizeof(*hdr), - hdr->caplen, hdr->len); - if (ret > 0) - pass++; - else if (ret == 0) - fail++; - else - return CMD_OK; - bpf_reset(); - } while (pcap_next_pkt() && (!has_limit || (++i < pkts))); - - rl_printf("bpf passes:%u fails:%u\n", pass, fail); - - pcap_reset_pkt(); - bpf_reset(); - - pass = fail = 0; - return CMD_OK; -} - -static int cmd_disassemble(char *line_string) -{ - bool single_line = false; - unsigned long line; - - if (!bpf_prog_loaded()) - return CMD_ERR; - if (strlen(line_string) > 0 && - (line = strtoul(line_string, NULL, 10)) < bpf_prog_len) - single_line = true; - if (single_line) - bpf_disasm(bpf_image[line], line); - else - bpf_disasm_all(bpf_image, bpf_prog_len); - - return CMD_OK; -} - -static int cmd_dump(char *dontcare) -{ - if (!bpf_prog_loaded()) - return CMD_ERR; - - bpf_dump_all(bpf_image, bpf_prog_len); - - return CMD_OK; -} - -static int cmd_quit(char *dontcare) -{ - return CMD_EX; -} - -static const struct shell_cmd cmds[] = { - { .name = "load", .func = cmd_load }, - { .name = "select", .func = cmd_select }, - { .name = "step", .func = cmd_step }, - { .name = "run", .func = cmd_run }, - { .name = "breakpoint", .func = cmd_breakpoint }, - { .name = "disassemble", .func = cmd_disassemble }, - { .name = "dump", .func = cmd_dump }, - { .name = "quit", .func = cmd_quit }, -}; - -static int execf(char *arg) -{ - char *cmd, *cont, *tmp = strdup(arg); - int i, ret = 0, len; - - cmd = strtok_r(tmp, " ", &cont); - if (cmd == NULL) - goto out; - len = strlen(cmd); - for (i = 0; i < array_size(cmds); i++) { - if (len != strlen(cmds[i].name)) - continue; - if (strncmp(cmds[i].name, cmd, len) == 0) { - ret = cmds[i].func(cont); - break; - } - } -out: - free(tmp); - return ret; -} - -static char *shell_comp_gen(const char *buf, int state) -{ - static int list_index, len; - - if (!state) { - list_index = 0; - len = strlen(buf); - } - - for (; list_index < array_size(cmds); ) { - const char *name = cmds[list_index].name; - - list_index++; - if (strncmp(name, buf, len) == 0) - return strdup(name); - } - - return NULL; -} - -static char **shell_completion(const char *buf, int start, int end) -{ - char **matches = NULL; - - if (start == 0) - matches = rl_completion_matches(buf, shell_comp_gen); - - return matches; -} - -static void intr_shell(int sig) -{ - if (rl_end) - rl_kill_line(-1, 0); - - rl_crlf(); - rl_refresh_line(0, 0); - rl_free_line_state(); -} - -static void init_shell(FILE *fin, FILE *fout) -{ - char file[128]; - - snprintf(file, sizeof(file), "%s/.bpf_dbg_history", getenv("HOME")); - read_history(file); - - rl_instream = fin; - rl_outstream = fout; - - rl_readline_name = "bpf_dbg"; - rl_terminal_name = getenv("TERM"); - - rl_catch_signals = 0; - rl_catch_sigwinch = 1; - - rl_attempted_completion_function = shell_completion; - - rl_bind_key('\t', rl_complete); - - rl_bind_key_in_map('\t', rl_complete, emacs_meta_keymap); - rl_bind_key_in_map('\033', rl_complete, emacs_meta_keymap); - - snprintf(file, sizeof(file), "%s/.bpf_dbg_init", getenv("HOME")); - rl_read_init_file(file); - - rl_prep_terminal(0); - rl_set_signals(); - - signal(SIGINT, intr_shell); -} - -static void exit_shell(FILE *fin, FILE *fout) -{ - char file[128]; - - snprintf(file, sizeof(file), "%s/.bpf_dbg_history", getenv("HOME")); - write_history(file); - - clear_history(); - rl_deprep_terminal(); - - try_close_pcap(); - - if (fin != stdin) - fclose(fin); - if (fout != stdout) - fclose(fout); -} - -static int run_shell_loop(FILE *fin, FILE *fout) -{ - char *buf; - - init_shell(fin, fout); - - while ((buf = readline("> ")) != NULL) { - int ret = execf(buf); - if (ret == CMD_EX) - break; - if (ret == CMD_OK && strlen(buf) > 0) - add_history(buf); - - free(buf); - } - - exit_shell(fin, fout); - return 0; -} - -int main(int argc, char **argv) -{ - FILE *fin = NULL, *fout = NULL; - - if (argc >= 2) - fin = fopen(argv[1], "r"); - if (argc >= 3) - fout = fopen(argv[2], "w"); - - return run_shell_loop(fin ? : stdin, fout ? : stdout); -}
diff --git a/tools/bpf/bpf_exp.l b/tools/bpf/bpf_exp.l deleted file mode 100644 index 4da8d05..0000000 --- a/tools/bpf/bpf_exp.l +++ /dev/null
@@ -1,198 +0,0 @@ -/* - * BPF asm code lexer - * - * This program is free software; you can distribute it and/or modify - * it under the terms of the GNU General Public License as published - * by the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * Syntax kept close to: - * - * Steven McCanne and Van Jacobson. 1993. The BSD packet filter: a new - * architecture for user-level packet capture. In Proceedings of the - * USENIX Winter 1993 Conference Proceedings on USENIX Winter 1993 - * Conference Proceedings (USENIX'93). USENIX Association, Berkeley, - * CA, USA, 2-2. - * - * Copyright 2013 Daniel Borkmann <borkmann@redhat.com> - * Licensed under the GNU General Public License, version 2.0 (GPLv2) - */ - -%{ - -#include <stdio.h> -#include <stdint.h> -#include <stdlib.h> -#include <string.h> - -#include <linux/filter.h> - -#include "bpf_exp.yacc.h" - -extern void yyerror(const char *str); - -%} - -%option align -%option ecs - -%option nounput -%option noreject -%option noinput -%option noyywrap - -%option 8bit -%option caseless -%option yylineno - -%% - -"ldb" { return OP_LDB; } -"ldh" { return OP_LDH; } -"ld" { return OP_LD; } -"ldi" { return OP_LDI; } -"ldx" { return OP_LDX; } -"ldxi" { return OP_LDXI; } -"ldxb" { return OP_LDXB; } -"st" { return OP_ST; } -"stx" { return OP_STX; } -"jmp" { return OP_JMP; } -"ja" { return OP_JMP; } -"jeq" { return OP_JEQ; } -"jneq" { return OP_JNEQ; } -"jne" { return OP_JNEQ; } -"jlt" { return OP_JLT; } -"jle" { return OP_JLE; } -"jgt" { return OP_JGT; } -"jge" { return OP_JGE; } -"jset" { return OP_JSET; } -"add" { return OP_ADD; } -"sub" { return OP_SUB; } -"mul" { return OP_MUL; } -"div" { return OP_DIV; } -"mod" { return OP_MOD; } -"neg" { return OP_NEG; } -"and" { return OP_AND; } -"xor" { return OP_XOR; } -"or" { return OP_OR; } -"lsh" { return OP_LSH; } -"rsh" { return OP_RSH; } -"ret" { return OP_RET; } -"tax" { return OP_TAX; } -"txa" { return OP_TXA; } - -"#"?("len") { return K_PKT_LEN; } - -"#"?("proto") { - yylval.number = SKF_AD_PROTOCOL; - return extension; - } -"#"?("type") { - yylval.number = SKF_AD_PKTTYPE; - return extension; - } -"#"?("poff") { - yylval.number = SKF_AD_PAY_OFFSET; - return extension; - } -"#"?("ifidx") { - yylval.number = SKF_AD_IFINDEX; - return extension; - } -"#"?("nla") { - yylval.number = SKF_AD_NLATTR; - return extension; - } -"#"?("nlan") { - yylval.number = SKF_AD_NLATTR_NEST; - return extension; - } -"#"?("mark") { - yylval.number = SKF_AD_MARK; - return extension; - } -"#"?("queue") { - yylval.number = SKF_AD_QUEUE; - return extension; - } -"#"?("hatype") { - yylval.number = SKF_AD_HATYPE; - return extension; - } -"#"?("rxhash") { - yylval.number = SKF_AD_RXHASH; - return extension; - } -"#"?("cpu") { - yylval.number = SKF_AD_CPU; - return extension; - } -"#"?("vlan_tci") { - yylval.number = SKF_AD_VLAN_TAG; - return extension; - } -"#"?("vlan_pr") { - yylval.number = SKF_AD_VLAN_TAG_PRESENT; - return extension; - } -"#"?("vlan_avail") { - yylval.number = SKF_AD_VLAN_TAG_PRESENT; - return extension; - } -"#"?("vlan_tpid") { - yylval.number = SKF_AD_VLAN_TPID; - return extension; - } -"#"?("rand") { - yylval.number = SKF_AD_RANDOM; - return extension; - } - -":" { return ':'; } -"," { return ','; } -"#" { return '#'; } -"%" { return '%'; } -"[" { return '['; } -"]" { return ']'; } -"(" { return '('; } -")" { return ')'; } -"x" { return 'x'; } -"a" { return 'a'; } -"+" { return '+'; } -"M" { return 'M'; } -"*" { return '*'; } -"&" { return '&'; } - -([0][x][a-fA-F0-9]+) { - yylval.number = strtoul(yytext, NULL, 16); - return number; - } -([0][b][0-1]+) { - yylval.number = strtol(yytext + 2, NULL, 2); - return number; - } -(([0])|([-+]?[1-9][0-9]*)) { - yylval.number = strtol(yytext, NULL, 10); - return number; - } -([0][0-7]+) { - yylval.number = strtol(yytext + 1, NULL, 8); - return number; - } -[a-zA-Z_][a-zA-Z0-9_]+ { - yylval.label = strdup(yytext); - return label; - } - -"/*"([^\*]|\*[^/])*"*/" { /* NOP */ } -";"[^\n]* { /* NOP */ } -^#.* { /* NOP */ } -[ \t]+ { /* NOP */ } -[ \n]+ { /* NOP */ } - -. { - printf("unknown character \'%s\'", yytext); - yyerror("lex unknown character"); - } - -%%
diff --git a/tools/bpf/bpf_exp.y b/tools/bpf/bpf_exp.y deleted file mode 100644 index dfb7254..0000000 --- a/tools/bpf/bpf_exp.y +++ /dev/null
@@ -1,668 +0,0 @@ -/* - * BPF asm code parser - * - * This program is free software; you can distribute it and/or modify - * it under the terms of the GNU General Public License as published - * by the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * Syntax kept close to: - * - * Steven McCanne and Van Jacobson. 1993. The BSD packet filter: a new - * architecture for user-level packet capture. In Proceedings of the - * USENIX Winter 1993 Conference Proceedings on USENIX Winter 1993 - * Conference Proceedings (USENIX'93). USENIX Association, Berkeley, - * CA, USA, 2-2. - * - * Copyright 2013 Daniel Borkmann <borkmann@redhat.com> - * Licensed under the GNU General Public License, version 2.0 (GPLv2) - */ - -%{ - -#include <stdio.h> -#include <string.h> -#include <stdint.h> -#include <stdlib.h> -#include <stdbool.h> -#include <unistd.h> -#include <errno.h> -#include <assert.h> -#include <linux/filter.h> - -#include "bpf_exp.yacc.h" - -enum jmp_type { JTL, JFL, JKL }; - -extern FILE *yyin; -extern int yylineno; -extern int yylex(void); -extern void yyerror(const char *str); - -extern void bpf_asm_compile(FILE *fp, bool cstyle); -static void bpf_set_curr_instr(uint16_t op, uint8_t jt, uint8_t jf, uint32_t k); -static void bpf_set_curr_label(char *label); -static void bpf_set_jmp_label(char *label, enum jmp_type type); - -%} - -%union { - char *label; - uint32_t number; -} - -%token OP_LDB OP_LDH OP_LD OP_LDX OP_ST OP_STX OP_JMP OP_JEQ OP_JGT OP_JGE -%token OP_JSET OP_ADD OP_SUB OP_MUL OP_DIV OP_AND OP_OR OP_XOR OP_LSH OP_RSH -%token OP_RET OP_TAX OP_TXA OP_LDXB OP_MOD OP_NEG OP_JNEQ OP_JLT OP_JLE OP_LDI -%token OP_LDXI - -%token K_PKT_LEN - -%token ':' ',' '[' ']' '(' ')' 'x' 'a' '+' 'M' '*' '&' '#' '%' - -%token extension number label - -%type <label> label -%type <number> extension -%type <number> number - -%% - -prog - : line - | prog line - ; - -line - : instr - | labelled_instr - ; - -labelled_instr - : labelled instr - ; - -instr - : ldb - | ldh - | ld - | ldi - | ldx - | ldxi - | st - | stx - | jmp - | jeq - | jneq - | jlt - | jle - | jgt - | jge - | jset - | add - | sub - | mul - | div - | mod - | neg - | and - | or - | xor - | lsh - | rsh - | ret - | tax - | txa - ; - -labelled - : label ':' { bpf_set_curr_label($1); } - ; - -ldb - : OP_LDB '[' 'x' '+' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_B | BPF_IND, 0, 0, $5); } - | OP_LDB '[' '%' 'x' '+' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_B | BPF_IND, 0, 0, $6); } - | OP_LDB '[' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_B | BPF_ABS, 0, 0, $3); } - | OP_LDB extension { - bpf_set_curr_instr(BPF_LD | BPF_B | BPF_ABS, 0, 0, - SKF_AD_OFF + $2); } - ; - -ldh - : OP_LDH '[' 'x' '+' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_H | BPF_IND, 0, 0, $5); } - | OP_LDH '[' '%' 'x' '+' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_H | BPF_IND, 0, 0, $6); } - | OP_LDH '[' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_H | BPF_ABS, 0, 0, $3); } - | OP_LDH extension { - bpf_set_curr_instr(BPF_LD | BPF_H | BPF_ABS, 0, 0, - SKF_AD_OFF + $2); } - ; - -ldi - : OP_LDI '#' number { - bpf_set_curr_instr(BPF_LD | BPF_IMM, 0, 0, $3); } - | OP_LDI number { - bpf_set_curr_instr(BPF_LD | BPF_IMM, 0, 0, $2); } - ; - -ld - : OP_LD '#' number { - bpf_set_curr_instr(BPF_LD | BPF_IMM, 0, 0, $3); } - | OP_LD K_PKT_LEN { - bpf_set_curr_instr(BPF_LD | BPF_W | BPF_LEN, 0, 0, 0); } - | OP_LD extension { - bpf_set_curr_instr(BPF_LD | BPF_W | BPF_ABS, 0, 0, - SKF_AD_OFF + $2); } - | OP_LD 'M' '[' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_MEM, 0, 0, $4); } - | OP_LD '[' 'x' '+' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_W | BPF_IND, 0, 0, $5); } - | OP_LD '[' '%' 'x' '+' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_W | BPF_IND, 0, 0, $6); } - | OP_LD '[' number ']' { - bpf_set_curr_instr(BPF_LD | BPF_W | BPF_ABS, 0, 0, $3); } - ; - -ldxi - : OP_LDXI '#' number { - bpf_set_curr_instr(BPF_LDX | BPF_IMM, 0, 0, $3); } - | OP_LDXI number { - bpf_set_curr_instr(BPF_LDX | BPF_IMM, 0, 0, $2); } - ; - -ldx - : OP_LDX '#' number { - bpf_set_curr_instr(BPF_LDX | BPF_IMM, 0, 0, $3); } - | OP_LDX K_PKT_LEN { - bpf_set_curr_instr(BPF_LDX | BPF_W | BPF_LEN, 0, 0, 0); } - | OP_LDX 'M' '[' number ']' { - bpf_set_curr_instr(BPF_LDX | BPF_MEM, 0, 0, $4); } - | OP_LDXB number '*' '(' '[' number ']' '&' number ')' { - if ($2 != 4 || $9 != 0xf) { - fprintf(stderr, "ldxb offset not supported!\n"); - exit(1); - } else { - bpf_set_curr_instr(BPF_LDX | BPF_MSH | BPF_B, 0, 0, $6); } } - | OP_LDX number '*' '(' '[' number ']' '&' number ')' { - if ($2 != 4 || $9 != 0xf) { - fprintf(stderr, "ldxb offset not supported!\n"); - exit(1); - } else { - bpf_set_curr_instr(BPF_LDX | BPF_MSH | BPF_B, 0, 0, $6); } } - ; - -st - : OP_ST 'M' '[' number ']' { - bpf_set_curr_instr(BPF_ST, 0, 0, $4); } - ; - -stx - : OP_STX 'M' '[' number ']' { - bpf_set_curr_instr(BPF_STX, 0, 0, $4); } - ; - -jmp - : OP_JMP label { - bpf_set_jmp_label($2, JKL); - bpf_set_curr_instr(BPF_JMP | BPF_JA, 0, 0, 0); } - ; - -jeq - : OP_JEQ '#' number ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, $3); } - | OP_JEQ 'x' ',' label ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_jmp_label($6, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 0); } - | OP_JEQ '%' 'x' ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 0); } - | OP_JEQ '#' number ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, $3); } - | OP_JEQ 'x' ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 0); } - | OP_JEQ '%' 'x' ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 0); } - ; - -jneq - : OP_JNEQ '#' number ',' label { - bpf_set_jmp_label($5, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, $3); } - | OP_JNEQ 'x' ',' label { - bpf_set_jmp_label($4, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 0); } - | OP_JNEQ '%' 'x' ',' label { - bpf_set_jmp_label($5, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 0); } - ; - -jlt - : OP_JLT '#' number ',' label { - bpf_set_jmp_label($5, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_K, 0, 0, $3); } - | OP_JLT 'x' ',' label { - bpf_set_jmp_label($4, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 0); } - | OP_JLT '%' 'x' ',' label { - bpf_set_jmp_label($5, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 0); } - ; - -jle - : OP_JLE '#' number ',' label { - bpf_set_jmp_label($5, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_K, 0, 0, $3); } - | OP_JLE 'x' ',' label { - bpf_set_jmp_label($4, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 0); } - | OP_JLE '%' 'x' ',' label { - bpf_set_jmp_label($5, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 0); } - ; - -jgt - : OP_JGT '#' number ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_K, 0, 0, $3); } - | OP_JGT 'x' ',' label ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_jmp_label($6, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 0); } - | OP_JGT '%' 'x' ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 0); } - | OP_JGT '#' number ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_K, 0, 0, $3); } - | OP_JGT 'x' ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 0); } - | OP_JGT '%' 'x' ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 0); } - ; - -jge - : OP_JGE '#' number ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_K, 0, 0, $3); } - | OP_JGE 'x' ',' label ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_jmp_label($6, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 0); } - | OP_JGE '%' 'x' ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 0); } - | OP_JGE '#' number ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_K, 0, 0, $3); } - | OP_JGE 'x' ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 0); } - | OP_JGE '%' 'x' ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 0); } - ; - -jset - : OP_JSET '#' number ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JSET | BPF_K, 0, 0, $3); } - | OP_JSET 'x' ',' label ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_jmp_label($6, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JSET | BPF_X, 0, 0, 0); } - | OP_JSET '%' 'x' ',' label ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_jmp_label($7, JFL); - bpf_set_curr_instr(BPF_JMP | BPF_JSET | BPF_X, 0, 0, 0); } - | OP_JSET '#' number ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JSET | BPF_K, 0, 0, $3); } - | OP_JSET 'x' ',' label { - bpf_set_jmp_label($4, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JSET | BPF_X, 0, 0, 0); } - | OP_JSET '%' 'x' ',' label { - bpf_set_jmp_label($5, JTL); - bpf_set_curr_instr(BPF_JMP | BPF_JSET | BPF_X, 0, 0, 0); } - ; - -add - : OP_ADD '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_ADD | BPF_K, 0, 0, $3); } - | OP_ADD 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_ADD | BPF_X, 0, 0, 0); } - | OP_ADD '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_ADD | BPF_X, 0, 0, 0); } - ; - -sub - : OP_SUB '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_SUB | BPF_K, 0, 0, $3); } - | OP_SUB 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_SUB | BPF_X, 0, 0, 0); } - | OP_SUB '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_SUB | BPF_X, 0, 0, 0); } - ; - -mul - : OP_MUL '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_MUL | BPF_K, 0, 0, $3); } - | OP_MUL 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_MUL | BPF_X, 0, 0, 0); } - | OP_MUL '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_MUL | BPF_X, 0, 0, 0); } - ; - -div - : OP_DIV '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_DIV | BPF_K, 0, 0, $3); } - | OP_DIV 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_DIV | BPF_X, 0, 0, 0); } - | OP_DIV '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_DIV | BPF_X, 0, 0, 0); } - ; - -mod - : OP_MOD '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_MOD | BPF_K, 0, 0, $3); } - | OP_MOD 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_MOD | BPF_X, 0, 0, 0); } - | OP_MOD '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_MOD | BPF_X, 0, 0, 0); } - ; - -neg - : OP_NEG { - bpf_set_curr_instr(BPF_ALU | BPF_NEG, 0, 0, 0); } - ; - -and - : OP_AND '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_AND | BPF_K, 0, 0, $3); } - | OP_AND 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_AND | BPF_X, 0, 0, 0); } - | OP_AND '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_AND | BPF_X, 0, 0, 0); } - ; - -or - : OP_OR '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_OR | BPF_K, 0, 0, $3); } - | OP_OR 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_OR | BPF_X, 0, 0, 0); } - | OP_OR '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_OR | BPF_X, 0, 0, 0); } - ; - -xor - : OP_XOR '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_XOR | BPF_K, 0, 0, $3); } - | OP_XOR 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_XOR | BPF_X, 0, 0, 0); } - | OP_XOR '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_XOR | BPF_X, 0, 0, 0); } - ; - -lsh - : OP_LSH '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_LSH | BPF_K, 0, 0, $3); } - | OP_LSH 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_LSH | BPF_X, 0, 0, 0); } - | OP_LSH '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_LSH | BPF_X, 0, 0, 0); } - ; - -rsh - : OP_RSH '#' number { - bpf_set_curr_instr(BPF_ALU | BPF_RSH | BPF_K, 0, 0, $3); } - | OP_RSH 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_RSH | BPF_X, 0, 0, 0); } - | OP_RSH '%' 'x' { - bpf_set_curr_instr(BPF_ALU | BPF_RSH | BPF_X, 0, 0, 0); } - ; - -ret - : OP_RET 'a' { - bpf_set_curr_instr(BPF_RET | BPF_A, 0, 0, 0); } - | OP_RET '%' 'a' { - bpf_set_curr_instr(BPF_RET | BPF_A, 0, 0, 0); } - | OP_RET 'x' { - bpf_set_curr_instr(BPF_RET | BPF_X, 0, 0, 0); } - | OP_RET '%' 'x' { - bpf_set_curr_instr(BPF_RET | BPF_X, 0, 0, 0); } - | OP_RET '#' number { - bpf_set_curr_instr(BPF_RET | BPF_K, 0, 0, $3); } - ; - -tax - : OP_TAX { - bpf_set_curr_instr(BPF_MISC | BPF_TAX, 0, 0, 0); } - ; - -txa - : OP_TXA { - bpf_set_curr_instr(BPF_MISC | BPF_TXA, 0, 0, 0); } - ; - -%% - -static int curr_instr = 0; -static struct sock_filter out[BPF_MAXINSNS]; -static char **labels, **labels_jt, **labels_jf, **labels_k; - -static void bpf_assert_max(void) -{ - if (curr_instr >= BPF_MAXINSNS) { - fprintf(stderr, "only max %u insns allowed!\n", BPF_MAXINSNS); - exit(1); - } -} - -static void bpf_set_curr_instr(uint16_t code, uint8_t jt, uint8_t jf, - uint32_t k) -{ - bpf_assert_max(); - out[curr_instr].code = code; - out[curr_instr].jt = jt; - out[curr_instr].jf = jf; - out[curr_instr].k = k; - curr_instr++; -} - -static void bpf_set_curr_label(char *label) -{ - bpf_assert_max(); - labels[curr_instr] = label; -} - -static void bpf_set_jmp_label(char *label, enum jmp_type type) -{ - bpf_assert_max(); - switch (type) { - case JTL: - labels_jt[curr_instr] = label; - break; - case JFL: - labels_jf[curr_instr] = label; - break; - case JKL: - labels_k[curr_instr] = label; - break; - } -} - -static int bpf_find_insns_offset(const char *label) -{ - int i, max = curr_instr, ret = -ENOENT; - - for (i = 0; i < max; i++) { - if (labels[i] && !strcmp(label, labels[i])) { - ret = i; - break; - } - } - - if (ret == -ENOENT) { - fprintf(stderr, "no such label \'%s\'!\n", label); - exit(1); - } - - return ret; -} - -static void bpf_stage_1_insert_insns(void) -{ - yyparse(); -} - -static void bpf_reduce_k_jumps(void) -{ - int i; - - for (i = 0; i < curr_instr; i++) { - if (labels_k[i]) { - int off = bpf_find_insns_offset(labels_k[i]); - out[i].k = (uint32_t) (off - i - 1); - } - } -} - -static uint8_t bpf_encode_jt_jf_offset(int off, int i) -{ - int delta = off - i - 1; - - if (delta < 0 || delta > 255) { - fprintf(stderr, "error: insn #%d jumps to insn #%d, " - "which is out of range\n", i, off); - exit(1); - } - return (uint8_t) delta; -} - -static void bpf_reduce_jt_jumps(void) -{ - int i; - - for (i = 0; i < curr_instr; i++) { - if (labels_jt[i]) { - int off = bpf_find_insns_offset(labels_jt[i]); - out[i].jt = bpf_encode_jt_jf_offset(off, i); - } - } -} - -static void bpf_reduce_jf_jumps(void) -{ - int i; - - for (i = 0; i < curr_instr; i++) { - if (labels_jf[i]) { - int off = bpf_find_insns_offset(labels_jf[i]); - out[i].jf = bpf_encode_jt_jf_offset(off, i); - } - } -} - -static void bpf_stage_2_reduce_labels(void) -{ - bpf_reduce_k_jumps(); - bpf_reduce_jt_jumps(); - bpf_reduce_jf_jumps(); -} - -static void bpf_pretty_print_c(void) -{ - int i; - - for (i = 0; i < curr_instr; i++) - printf("{ %#04x, %2u, %2u, %#010x },\n", out[i].code, - out[i].jt, out[i].jf, out[i].k); -} - -static void bpf_pretty_print(void) -{ - int i; - - printf("%u,", curr_instr); - for (i = 0; i < curr_instr; i++) - printf("%u %u %u %u,", out[i].code, - out[i].jt, out[i].jf, out[i].k); - printf("\n"); -} - -static void bpf_init(void) -{ - memset(out, 0, sizeof(out)); - - labels = calloc(BPF_MAXINSNS, sizeof(*labels)); - assert(labels); - labels_jt = calloc(BPF_MAXINSNS, sizeof(*labels_jt)); - assert(labels_jt); - labels_jf = calloc(BPF_MAXINSNS, sizeof(*labels_jf)); - assert(labels_jf); - labels_k = calloc(BPF_MAXINSNS, sizeof(*labels_k)); - assert(labels_k); -} - -static void bpf_destroy_labels(void) -{ - int i; - - for (i = 0; i < curr_instr; i++) { - free(labels_jf[i]); - free(labels_jt[i]); - free(labels_k[i]); - free(labels[i]); - } -} - -static void bpf_destroy(void) -{ - bpf_destroy_labels(); - free(labels_jt); - free(labels_jf); - free(labels_k); - free(labels); -} - -void bpf_asm_compile(FILE *fp, bool cstyle) -{ - yyin = fp; - - bpf_init(); - bpf_stage_1_insert_insns(); - bpf_stage_2_reduce_labels(); - bpf_destroy(); - - if (cstyle) - bpf_pretty_print_c(); - else - bpf_pretty_print(); - - if (fp != stdin) - fclose(yyin); -} - -void yyerror(const char *str) -{ - fprintf(stderr, "error: %s at line %d\n", str, yylineno); - exit(1); -}
diff --git a/tools/bpf/bpf_jit_disasm.c b/tools/bpf/bpf_jit_disasm.c deleted file mode 100644 index 5ab8f80..0000000 --- a/tools/bpf/bpf_jit_disasm.c +++ /dev/null
@@ -1,334 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF JIT image disassembler - * - * Disassembles BPF JIT compiler emitted opcodes back to asm insn's for - * debugging or verification purposes. - * - * To get the disassembly of the JIT code, do the following: - * - * 1) `echo 2 > /proc/sys/net/core/bpf_jit_enable` - * 2) Load a BPF filter (e.g. `tcpdump -p -n -s 0 -i eth1 host 192.168.20.0/24`) - * 3) Run e.g. `bpf_jit_disasm -o` to read out the last JIT code - * - * Copyright 2013 Daniel Borkmann <borkmann@redhat.com> - */ - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <unistd.h> -#include <string.h> -#include <bfd.h> -#include <dis-asm.h> -#include <regex.h> -#include <fcntl.h> -#include <sys/klog.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <limits.h> -#include <tools/dis-asm-compat.h> - -#define CMD_ACTION_SIZE_BUFFER 10 -#define CMD_ACTION_READ_ALL 3 - -static void get_exec_path(char *tpath, size_t size) -{ - char *path; - ssize_t len; - - snprintf(tpath, size, "/proc/%d/exe", (int) getpid()); - tpath[size - 1] = 0; - - path = strdup(tpath); - assert(path); - - len = readlink(path, tpath, size); - if (len < 0) - len = 0; - tpath[len] = 0; - - free(path); -} - -static void get_asm_insns(uint8_t *image, size_t len, int opcodes) -{ - int count, i, pc = 0; - char tpath[PATH_MAX]; - struct disassemble_info info; - disassembler_ftype disassemble; - bfd *bfdf; - - memset(tpath, 0, sizeof(tpath)); - get_exec_path(tpath, sizeof(tpath)); - - bfdf = bfd_openr(tpath, NULL); - assert(bfdf); - assert(bfd_check_format(bfdf, bfd_object)); - - init_disassemble_info_compat(&info, stdout, - (fprintf_ftype) fprintf, - fprintf_styled); - info.arch = bfd_get_arch(bfdf); - info.mach = bfd_get_mach(bfdf); - info.buffer = image; - info.buffer_length = len; - - disassemble_init_for_target(&info); - -#ifdef DISASM_FOUR_ARGS_SIGNATURE - disassemble = disassembler(info.arch, - bfd_big_endian(bfdf), - info.mach, - bfdf); -#else - disassemble = disassembler(bfdf); -#endif - assert(disassemble); - - do { - printf("%4x:\t", pc); - - count = disassemble(pc, &info); - - if (opcodes) { - printf("\n\t"); - for (i = 0; i < count; ++i) - printf("%02x ", (uint8_t) image[pc + i]); - } - printf("\n"); - - pc += count; - } while(count > 0 && pc < len); - - bfd_close(bfdf); -} - -static char *get_klog_buff(unsigned int *klen) -{ - int ret, len; - char *buff; - - len = klogctl(CMD_ACTION_SIZE_BUFFER, NULL, 0); - if (len < 0) - return NULL; - - buff = malloc(len); - if (!buff) - return NULL; - - ret = klogctl(CMD_ACTION_READ_ALL, buff, len); - if (ret < 0) { - free(buff); - return NULL; - } - - *klen = ret; - return buff; -} - -static char *get_flog_buff(const char *file, unsigned int *klen) -{ - int fd, ret, len; - struct stat fi; - char *buff; - - fd = open(file, O_RDONLY); - if (fd < 0) - return NULL; - - ret = fstat(fd, &fi); - if (ret < 0 || !S_ISREG(fi.st_mode)) - goto out; - - len = fi.st_size + 1; - buff = malloc(len); - if (!buff) - goto out; - - memset(buff, 0, len); - ret = read(fd, buff, len - 1); - if (ret <= 0) - goto out_free; - - close(fd); - *klen = ret; - return buff; -out_free: - free(buff); -out: - close(fd); - return NULL; -} - -static char *get_log_buff(const char *file, unsigned int *klen) -{ - return file ? get_flog_buff(file, klen) : get_klog_buff(klen); -} - -static void put_log_buff(char *buff) -{ - free(buff); -} - -static uint8_t *get_last_jit_image(char *haystack, size_t hlen, - unsigned int *ilen) -{ - char *ptr, *pptr, *tmp; - off_t off = 0; - unsigned int proglen; - int ret, flen, pass, ulen = 0; - regmatch_t pmatch[1]; - unsigned long base; - regex_t regex; - uint8_t *image; - - if (hlen == 0) - return NULL; - - ret = regcomp(®ex, "flen=[[:alnum:]]+ proglen=[[:digit:]]+ " - "pass=[[:digit:]]+ image=[[:xdigit:]]+", REG_EXTENDED); - assert(ret == 0); - - ptr = haystack; - memset(pmatch, 0, sizeof(pmatch)); - - while (1) { - ret = regexec(®ex, ptr, 1, pmatch, 0); - if (ret == 0) { - ptr += pmatch[0].rm_eo; - off += pmatch[0].rm_eo; - assert(off < hlen); - } else - break; - } - - ptr = haystack + off - (pmatch[0].rm_eo - pmatch[0].rm_so); - ret = sscanf(ptr, "flen=%d proglen=%u pass=%d image=%lx", - &flen, &proglen, &pass, &base); - if (ret != 4) { - regfree(®ex); - return NULL; - } - if (proglen > 1000000) { - printf("proglen of %u too big, stopping\n", proglen); - return NULL; - } - - image = malloc(proglen); - if (!image) { - printf("Out of memory\n"); - return NULL; - } - memset(image, 0, proglen); - - tmp = ptr = haystack + off; - while ((ptr = strtok(tmp, "\n")) != NULL && ulen < proglen) { - tmp = NULL; - if (!strstr(ptr, "JIT code")) - continue; - pptr = ptr; - while ((ptr = strstr(pptr, ":"))) - pptr = ptr + 1; - ptr = pptr; - do { - image[ulen++] = (uint8_t) strtoul(pptr, &pptr, 16); - if (ptr == pptr) { - ulen--; - break; - } - if (ulen >= proglen) - break; - ptr = pptr; - } while (1); - } - - assert(ulen == proglen); - printf("%u bytes emitted from JIT compiler (pass:%d, flen:%d)\n", - proglen, pass, flen); - printf("%lx + <x>:\n", base); - - regfree(®ex); - *ilen = ulen; - return image; -} - -static void usage(void) -{ - printf("Usage: bpf_jit_disasm [...]\n"); - printf(" -o Also display related opcodes (default: off).\n"); - printf(" -O <file> Write binary image of code to file, don't disassemble to stdout.\n"); - printf(" -f <file> Read last image dump from file or stdin (default: klog).\n"); - printf(" -h Display this help.\n"); -} - -int main(int argc, char **argv) -{ - unsigned int len, klen, opt, opcodes = 0; - char *kbuff, *file = NULL; - char *ofile = NULL; - int ofd; - ssize_t nr; - uint8_t *pos; - uint8_t *image = NULL; - - while ((opt = getopt(argc, argv, "of:O:")) != -1) { - switch (opt) { - case 'o': - opcodes = 1; - break; - case 'O': - ofile = optarg; - break; - case 'f': - file = optarg; - break; - default: - usage(); - return -1; - } - } - - bfd_init(); - - kbuff = get_log_buff(file, &klen); - if (!kbuff) { - fprintf(stderr, "Could not retrieve log buffer!\n"); - return -1; - } - - image = get_last_jit_image(kbuff, klen, &len); - if (!image) { - fprintf(stderr, "No JIT image found!\n"); - goto done; - } - if (!ofile) { - get_asm_insns(image, len, opcodes); - goto done; - } - - ofd = open(ofile, O_WRONLY | O_CREAT | O_TRUNC, DEFFILEMODE); - if (ofd < 0) { - fprintf(stderr, "Could not open file %s for writing: ", ofile); - perror(NULL); - goto done; - } - pos = image; - do { - nr = write(ofd, pos, len); - if (nr < 0) { - fprintf(stderr, "Could not write data to %s: ", ofile); - perror(NULL); - goto done; - } - len -= nr; - pos += nr; - } while (len); - close(ofd); - -done: - put_log_buff(kbuff); - free(image); - return 0; -}
diff --git a/tools/bpf/bpftool/Documentation/bpftool-btf.rst b/tools/bpf/bpftool/Documentation/bpftool-btf.rst index cf75a7f..c6dc445 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-btf.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-btf.rst
@@ -16,7 +16,7 @@ **bpftool** [*OPTIONS*] **btf** *COMMAND* -*OPTIONS* := { |COMMON_OPTIONS| | { **-B** | **--base-btf** } } +*OPTIONS* := { |COMMON_OPTIONS| | [{ **-B** | **--base-btf**} *FILE*]... } *COMMANDS* := { **dump** | **help** } @@ -87,7 +87,10 @@ objects for kernel modules. To avoid duplicating all kernel symbols required by modules, BTF objects for modules are "split", they are built incrementally on top of the kernel (vmlinux) BTF object. So the - base BTF reference should usually point to the kernel BTF. + base BTF reference should usually point to the kernel BTF. Multiple + base BTF objects can be passed, where the first is assumed to be the + root BTF, followed by split BTF based upon it, followed by split + BTF based upon the first split BTF and so on. When the main BTF object to process (for example, the module BTF to dump) is passed as a *FILE*, bpftool attempts to autodetect the path
diff --git a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst index e818559..d1b8193 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
@@ -45,7 +45,7 @@ | **cgroup_unix_recvmsg** | **cgroup_sysctl** | | **cgroup_getsockopt** | **cgroup_setsockopt** | | **cgroup_inet_sock_release** } -| *ATTACH_FLAGS* := { **multi** | **override** } +| *ATTACH_FLAGS* := { [ **multi** | **override** ] [ **preorder** ] } DESCRIPTION =========== @@ -75,20 +75,27 @@ Attach program *PROG* to the cgroup *CGROUP* with attach type *ATTACH_TYPE* and optional *ATTACH_FLAGS*. - *ATTACH_FLAGS* can be one of: **override** if a sub-cgroup installs some + *ATTACH_FLAGS* can include: **override** if a sub-cgroup installs some bpf program, the program in this cgroup yields to sub-cgroup program; **multi** if a sub-cgroup installs some bpf program, that cgroup program - gets run in addition to the program in this cgroup. + gets run in addition to the program in this cgroup; + **preorder** requests ancestor-to-descendant execution for this program, + before non-preorder programs, which execute descendants-to-ancestors + across the cgroup hierarchy. Note that **preorder** alone does not enable + multi-program attachment; specify **multi** together with **preorder** to + attach multiple programs. - Only one program is allowed to be attached to a cgroup with no attach flags - or the **override** flag. Attaching another program will release old - program and attach the new one. + Only one program is allowed to be attached to a cgroup unless the + **multi** flag is specified. Without **multi**, attaching another program + replaces the existing program, provided the **override** setting matches. Multiple programs are allowed to be attached to a cgroup with **multi**. - They are executed in FIFO order (those that were attached first, run - first). + Programs marked with **preorder** are placed before non-preorder programs + in the effective program array. Within each ordering class at the same + cgroup level, attachment order is preserved. - Non-default *ATTACH_FLAGS* are supported by kernel version 4.14 and later. + **multi** and **override** are supported by kernel version 4.14 and later. + **preorder** was introduced upstream in Linux 6.15. *ATTACH_TYPE* can be one of:
diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst index 5daf3de..e4ed7e7 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-map.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst
@@ -120,7 +120,8 @@ character ('.'), which is reserved for future extensions of *bpffs*. bpftool map event_pipe *MAP* [cpu *N* index *M*] - Read events from a **BPF_MAP_TYPE_PERF_EVENT_ARRAY** map. + Read events from a **BPF_MAP_TYPE_PERF_EVENT_ARRAY** or + **BPF_MAP_TYPE_RINGBUF** map. Install perf rings into a perf event array map and dump output of any **bpf_perf_event_output**\ () call in the kernel. By default read the @@ -134,6 +135,17 @@ existing ring. Any other application will stop receiving events if it installed its rings earlier. + For a ring buffer map, consume records submitted by BPF programs, including + records already queued before the command starts. **cpu** and **index** + are not supported. Each record is printed in full, including embedded zero + bytes. Plain output reports the record size followed by hexadecimal + bytes; JSON output contains **size** and **data** fields, with **data** an + array of byte values. Ring buffer records have no implicit CPU or timestamp. + + Consuming a ring buffer advances its shared consumer position, so this + command must not run alongside another consumer of the same map. + **BPF_MAP_TYPE_USER_RINGBUF** maps are not supported. + bpftool map peek *MAP* Peek next value in the queue or stack.
diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 90fa2a4..8dbd34e 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
@@ -18,7 +18,8 @@ *OPTIONS* := { |COMMON_OPTIONS| | { **-f** | **--bpffs** } | { **-m** | **--mapcompat** } | { **-n** | **--nomount** } | -{ **-L** | **--use-loader** } | [ { **-S** | **--sign** } **-k** <private_key.pem> **-i** <certificate.x509> ] } +{ **-L** | **--use-loader** } | [ { **-S** | **--sign** } **-k** <private_key.pem> **-i** <certificate.x509> ] | +{ **-w** | **--wait** } } *COMMANDS* := { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** | @@ -186,6 +187,12 @@ error messages to the standard error stream. This facility should be used only for debugging purposes. + By default, bpftool prints the output buffered so far and exits. With + **-w** or **--wait**, it keeps printing new output as the program produces + it, until the program is unloaded or <Ctrl+C> is hit. Waiting requires a + kernel that supports opening a stream as a file descriptor, introduced + upstream in Linux 7.4. + bpftool prog run *PROG* data_in *FILE* [data_out *FILE* [data_size_out *L*]] [ctx_in *FILE* [ctx_out *FILE* [ctx_size_out *M*]]] [repeat *N*] Run BPF program *PROG* in the kernel testing infrastructure for BPF, meaning that the program works on the data and context provided by the @@ -217,7 +224,14 @@ bpftool prog profile *PROG* [duration *DURATION*] *METRICs* Profile *METRICs* for bpf program *PROG* for *DURATION* seconds or until user hits <Ctrl+C>. *DURATION* is optional. If *DURATION* is not specified, - the profiling will run up to **UINT_MAX** seconds. + the profiling will run up to **UINT_MAX** seconds. Selected metrics form a + perf event group on each CPU so that ratios use counters scheduled over the + same intervals. Plain output scales each per-CPU metric value to correct + for perf event multiplexing. When **cycles** is selected, it also reports + cycles per program run. + + In JSON output, **value** is raw, **value_scaled** is scaled, and + **run_cnt** is the total number of program runs. bpftool prog help Print short help message. @@ -260,6 +274,11 @@ Path to the X.509 certificate file in PEM or DER format, required when signing. +-w, --wait + When dumping a program stream with **bpftool prog tracelog** { **stdout** | + **stderr** } *PROG*, wait for new output instead of exiting once the + buffered output has been printed. + EXAMPLES ======== **# bpftool prog show** @@ -360,7 +379,7 @@ :: 51397 run_cnt - 40176203 cycles (83.05%) + 40176203 cycles # 781.68 cycles per run (83.05%) 42518139 instructions # 1.06 insns per cycle (83.39%) 123 llc_misses # 2.89 LLC misses per million insns (83.15%)
diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 75cbcb5..a28af31 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool
@@ -50,12 +50,19 @@ command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) ) } -# Takes map type and adds matching map ids to the list of suggestions. -_bpftool_get_map_ids_for_type() +# Takes map types and prints a pattern matching their JSON type fields. +_bpftool_build_type_pattern() { - local type="$1" + local IFS='|' + printf '"type": "(%s)"\n' "$*" +} + +# Takes map types and adds matching map ids to the list of suggestions. +_bpftool_get_map_ids_for_types() +{ + local types=$(_bpftool_build_type_pattern "$@") COMPREPLY+=( $( compgen -W "$( bpftool -jp map 2>&1 | \ - command grep -C2 "$type" | \ + command grep -C2 -E "$types" | \ command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) ) } @@ -65,12 +72,12 @@ command sed -n 's/.*"name": \(.*\),$/\1/p' )" -- "$cur" ) ) } -# Takes map type and adds matching map names to the list of suggestions. -_bpftool_get_map_names_for_type() +# Takes map types and adds matching map names to the list of suggestions. +_bpftool_get_map_names_for_types() { - local type="$1" + local types=$(_bpftool_build_type_pattern "$@") COMPREPLY+=( $( compgen -W "$( bpftool -jp map 2>&1 | \ - command grep -C2 "$type" | \ + command grep -C2 -E "$types" | \ command sed -n 's/.*"name": \(.*\),$/\1/p' )" -- "$cur" ) ) } @@ -139,7 +146,7 @@ local keyword idx ref="" for (( idx=3; idx < ${#words[@]}-1; idx++ )); do case "${words[$((idx-2))]}" in - lookup|update) + lookup|update|event_pipe) keyword=${words[$((idx-1))]} ref=${words[$((idx))]} ;; @@ -177,10 +184,10 @@ if [[ $value -eq 0 ]]; then case "$command" in push) - _bpftool_get_map_ids_for_type stack + _bpftool_get_map_ids_for_types stack ;; enqueue) - _bpftool_get_map_ids_for_type queue + _bpftool_get_map_ids_for_types queue ;; *) _bpftool_get_map_ids @@ -223,10 +230,10 @@ if [[ $value -eq 0 ]]; then case "$command" in push) - _bpftool_get_map_names_for_type stack + _bpftool_get_map_names_for_types stack ;; enqueue) - _bpftool_get_map_names_for_type queue + _bpftool_get_map_names_for_types queue ;; *) _bpftool_get_map_names @@ -262,7 +269,7 @@ # Deal with options if [[ ${words[cword]} == -* ]]; then local c='--version --json --pretty --bpffs --mapcompat --debug \ - --use-loader --base-btf --sign -i -k' + --use-loader --base-btf --sign -i -k --wait' COMPREPLY=( $( compgen -W "$c" -- "$cur" ) ) return 0 fi @@ -617,10 +624,10 @@ COMPREPLY=( $( compgen -W "$STRUCT_OPS_TYPE" -- "$cur" ) ) ;; id) - _bpftool_get_map_ids_for_type struct_ops + _bpftool_get_map_ids_for_types struct_ops ;; name) - _bpftool_get_map_names_for_type struct_ops + _bpftool_get_map_names_for_types struct_ops ;; esac return 0 @@ -679,14 +686,13 @@ id) case "$command" in peek) - _bpftool_get_map_ids_for_type stack - _bpftool_get_map_ids_for_type queue + _bpftool_get_map_ids_for_types stack queue ;; pop) - _bpftool_get_map_ids_for_type stack + _bpftool_get_map_ids_for_types stack ;; dequeue) - _bpftool_get_map_ids_for_type queue + _bpftool_get_map_ids_for_types queue ;; *) _bpftool_get_map_ids @@ -697,14 +703,13 @@ name) case "$command" in peek) - _bpftool_get_map_names_for_type stack - _bpftool_get_map_names_for_type queue + _bpftool_get_map_names_for_types stack queue ;; pop) - _bpftool_get_map_names_for_type stack + _bpftool_get_map_names_for_types stack ;; dequeue) - _bpftool_get_map_names_for_type queue + _bpftool_get_map_names_for_types queue ;; *) _bpftool_get_map_names @@ -878,11 +883,11 @@ return 0 ;; id) - _bpftool_get_map_ids_for_type perf_event_array + _bpftool_get_map_ids_for_types perf_event_array ringbuf return 0 ;; name) - _bpftool_get_map_names_for_type perf_event_array + _bpftool_get_map_names_for_types perf_event_array ringbuf return 0 ;; cpu) @@ -892,7 +897,9 @@ return 0 ;; *) - _bpftool_once_attr 'cpu index' + if [[ $(_bpftool_map_guess_map_type) != ringbuf ]]; then + _bpftool_once_attr 'cpu index' + fi return 0 ;; esac @@ -1057,7 +1064,6 @@ attach|detach) local BPFTOOL_CGROUP_ATTACH_TYPES="$(bpftool feature list_builtins attach_types 2>/dev/null | \ grep '^cgroup_')" - local ATTACH_FLAGS='multi override' # Check for $prev = $command first if [ $prev = $command ]; then _filedir @@ -1087,7 +1093,8 @@ # "id|pinned|tag|name" (we already checked for # that). This should only leave the case when # we need attach flags for "attach" commamnd. - _bpftool_one_of_list "$ATTACH_FLAGS" + _bpftool_one_of_list 'multi override' + _bpftool_once_attr 'preorder' fi return 0 ;;
diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index c958902..abd661d 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c
@@ -29,6 +29,7 @@ #define MAX_ROOT_IDS 16 #define MAX_BTF_FILES 64 +#define MAX_LOC_PARAM_WORDS 8 static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_UNKN] = "UNKNOWN", @@ -51,6 +52,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; struct sort_datum { @@ -117,6 +121,238 @@ static int btf_kind_safe(int kind) return kind <= BTF_KIND_MAX ? kind : BTF_KIND_UNKN; } +static void btf_loc_param_reg_str(__u32 reg, char *str, size_t sz) +{ + if (reg == BTF_LOC_PARAM_FBREG) + snprintf(str, sz, "fbreg"); + else + snprintf(str, sz, "reg%u", reg); +} + +static void btf_loc_param_raw_str(const struct btf_loc_param *p, __u32 vlen, + char *str, size_t sz) +{ + __u32 i, nr_words = min(vlen, (__u32)MAX_LOC_PARAM_WORDS); + size_t off = 0; + + if (!sz) + return; + + off += snprintf(str + off, sz - off, "raw=["); + for (i = 0; i < nr_words && off < sz; i++) + off += snprintf(str + off, sz - off, "%s0x%08x", + i ? ", " : "", p->values[i]); + if (vlen > nr_words && off < sz) + off += snprintf(str + off, sz - off, ", ..."); + if (off < sz) + snprintf(str + off, sz - off, "]"); +} + +static void btf_loc_param_str(const struct btf_type *t, char *str, size_t sz) +{ + const char *op = "", *prefix = ""; + const struct btf_loc_param *p; + __u32 i = 0, vlen; + __u64 value; + __u32 value_size; + bool negative = false; + char regs[32] = {}; + char num[32] = {}; + + if (!t || !btf_is_loc_param(t)) { + snprintf(str, sz, "<invalid>"); + return; + } + + p = btf_loc_param(t); + vlen = btf_vlen(t); + + if (p->flags & BTF_LOC_PARAM_REG) { + __u32 nregs = (p->flags == BTF_LOC_PARAM_REG) ? vlen : 1; + + if (nregs > vlen) { + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + + switch (nregs) { + case 2: + btf_loc_param_reg_str(p->values[0], regs, sizeof(regs)); + snprintf(regs + strlen(regs), sizeof(regs) - strlen(regs), + ", "); + btf_loc_param_reg_str(p->values[1], + regs + strlen(regs), + sizeof(regs) - strlen(regs)); + break; + case 1: + btf_loc_param_reg_str(p->values[0], regs, sizeof(regs)); + break; + default: + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + i += nregs; + } + if (p->flags & (BTF_LOC_PARAM_CONST|BTF_LOC_PARAM_OFFSET)) { + switch (vlen - i) { + case 1: + value_size = sizeof(p->values[0]); + value = p->values[i]; + break; + case 2: + value_size = 2 * sizeof(p->values[0]); + value = ((__u64)p->values[i + 1] << 32) | p->values[i]; + break; + default: + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + i = vlen; + if (p->flags & BTF_LOC_PARAM_SIGNED) { + /* + * size describes the represented parameter, so it + * describes a constant's signed width. An offset + * is determined by its value words after the register + * number. + */ + __u32 size = p->flags & BTF_LOC_PARAM_OFFSET ? + value_size : t->size; + __u32 bits = size * 8; + + /* + * Since we represent constant values in hex, we + * need to determine if the value is negative so + * we can prepend a "-", and also fix the value + * to be positive so we can have - 0x<value>. + */ + if (size && size <= sizeof(value)) { + if (size < sizeof(value)) + value &= (1ULL << bits) - 1; + negative = value & (1ULL << (bits - 1)); + if (negative) + value = size == sizeof(value) ? -value : + (1ULL << bits) - value; + } + } + if (p->flags & BTF_LOC_PARAM_ADDR) + prefix = "address "; + else if (p->flags & BTF_LOC_PARAM_CONST) + prefix = "const "; + snprintf(num, sizeof(num), "0x%llx", + (unsigned long long)value); + } + if (i != vlen) { + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + if (num[0]) + op = regs[0] ? (negative ? " - " : " + ") : negative ? "-" : ""; + + snprintf(str, sz, "%s%s%s%s%s%s", + p->flags & BTF_LOC_PARAM_DEREF ? "*(" : "", + prefix, + regs, + op, + num, + p->flags & BTF_LOC_PARAM_DEREF ? ")" : ""); +} + +static bool btf_locsec_append(char *str, size_t sz, size_t *off, + const char *suffix) +{ + static const char nospace_suffix[] = "..."; + bool space_left = true; + size_t left = 0, len; + + if (*off >= sz || sz < sizeof(nospace_suffix)) + return false; + left = sz - *off; + + /* + * Copy as much of the string as we can in remaining space, and + * append "..." if we would have overrun. + */ + len = strlen(suffix) + 1; + if (len > left) { + len = left; + space_left = false; + } + + memcpy(str + *off, suffix, len); + *off += len - 1; + + if (!space_left) + memcpy(str + sz - sizeof(nospace_suffix), nospace_suffix, sizeof(nospace_suffix)); + + return space_left; +} + +static void btf_locsec_func_str(const struct btf *btf, + const struct btf_loc *loc, char *str, size_t sz) +{ + const struct btf_type *func, *func_proto, *loc_proto; + const struct btf_param *params; + const __u32 *loc_params; + const char *name; + __u32 i, vlen; + size_t off = 0; + + if (!sz) + return; + + str[0] = '\0'; + func = btf__type_by_id(btf, loc->func); + if (!func || !btf_is_func(func)) + goto invalid; + + name = btf_str(btf, func->name_off); + func_proto = btf__type_by_id(btf, func->type); + loc_proto = btf__type_by_id(btf, loc->loc_proto); + if (!func_proto || !btf_is_func_proto(func_proto) || + !loc_proto || !btf_is_loc_proto(loc_proto) || + btf_vlen(func_proto) != btf_vlen(loc_proto)) + goto invalid; + + params = (const void *)(func_proto + 1); + loc_params = btf_loc_proto_params(loc_proto); + vlen = btf_vlen(func_proto); + if (!btf_locsec_append(str, sz, &off, name) || + !btf_locsec_append(str, sz, &off, "(")) + return; + for (i = 0; i < vlen; i++) { + const struct btf_type *param_loc; + char param_str[256] = {}; + + if (!params[i].type) { + /* Handle varargs func proto, must be last parameter */ + if (i != vlen - 1) + goto invalid; + if (!btf_locsec_append(str, sz, &off, i ? ", " : "") || + !btf_locsec_append(str, sz, &off, "...")) + return; + break; + } else if (loc_params[i]) { + param_loc = btf__type_by_id(btf, loc_params[i]); + btf_loc_param_str(param_loc, param_str, sizeof(param_str)); + } else { + snprintf(param_str, sizeof(param_str), "<unavailable>"); + } + + if (!btf_locsec_append(str, sz, &off, i ? ", " : "") || + !btf_locsec_append(str, sz, &off, + btf_str(btf, params[i].name_off)) || + !btf_locsec_append(str, sz, &off, " [") || + !btf_locsec_append(str, sz, &off, param_str) || + !btf_locsec_append(str, sz, &off, "]")) + return; + } + (void) btf_locsec_append(str, sz, &off, ")"); + return; + +invalid: + snprintf(str, sz, "<invalid>"); +} + static int dump_btf_type(const struct btf *btf, __u32 id, const struct btf_type *t) { @@ -415,6 +651,99 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } break; } + case BTF_KIND_LOC_PARAM: { + const struct btf_loc_param *p = btf_loc_param(t); + __u32 vlen = btf_vlen(t); + char param_str[256] = {}; + + btf_loc_param_str(t, param_str, sizeof(param_str)); + + if (json_output) { + jsonw_uint_field(w, "size", t->size); + jsonw_uint_field(w, "flags", p->flags); + jsonw_uint_field(w, "vlen", vlen); + jsonw_string_field(w, "values", param_str); + } else { + printf(" size=%u flags=0x%x vlen=%u values='%s'", + t->size, p->flags, vlen, param_str); + } + break; + } + case BTF_KIND_LOC_PROTO: { + __u32 *params = btf_loc_proto_params(t); + __u32 i, vlen = btf_vlen(t); + + if (json_output) { + jsonw_uint_field(w, "vlen", vlen); + jsonw_name(w, "params"); + jsonw_start_array(w); + } else { + printf(" vlen=%u", vlen); + } + + for (i = 0; i < vlen; i++, params++) { + const struct btf_type *p; + char param_str[256] = {}; + + if (*params) { + p = btf__type_by_id(btf, *params); + btf_loc_param_str(p, param_str, sizeof(param_str)); + } else { + snprintf(param_str, sizeof(param_str), "<unavailable>"); + } + if (json_output) { + jsonw_start_object(w); + jsonw_uint_field(w, "type_id", *params); + jsonw_string_field(w, "value", param_str); + jsonw_end_object(w); + } else { + printf("\n\ttype_id=%u value='%s'", *params, param_str); + } + } + if (json_output) + jsonw_end_array(w); + break; + } + + case BTF_KIND_LOCSEC: { + struct btf_loc *locs = btf_locsec_locs(t); + __u32 i, vlen = btf_vlen(t); + + if (json_output) { + jsonw_uint_field(w, "vlen", vlen); + jsonw_name(w, "locs"); + jsonw_start_array(w); + } else { + printf(" vlen=%u", vlen); + } + + for (i = 0; i < vlen; i++, locs++) { + const struct btf_type *func; + const char *name = "<invalid>"; + char func_str[1024] = {}; + + func = btf__type_by_id(btf, locs->func); + if (func && btf_is_func(func)) + name = btf_str(btf, func->name_off); + btf_locsec_func_str(btf, locs, func_str, sizeof(func_str)); + + if (json_output) { + jsonw_start_object(w); + jsonw_uint_field(w, "func_type_id", locs->func); + jsonw_string_field(w, "name", name); + jsonw_string_field(w, "func", func_str); + jsonw_uint_field(w, "loc_proto_type_id", locs->loc_proto); + jsonw_uint_field(w, "offset", locs->offset); + jsonw_end_object(w); + } else { + printf("\n\tfunc='%s' func_type_id=%u loc_proto_type_id=%u offset=%u", + func_str, locs->func, locs->loc_proto, locs->offset); + } + } + if (json_output) + jsonw_end_array(w); + break; + } default: break; } @@ -805,9 +1134,13 @@ static int dump_btf_c(const struct btf *btf, if (sort_dump) datums = sort_btf_c(btf); - for (i = 1; i < cnt; i++) { + for (i = 0; i < cnt; i++) { int idx = datums ? datums[i].index : i; + /* type ID 0 is void, skip it */ + if (idx == 0) + continue; + err = btf_dump__dump_type(d, idx); if (err) goto done; @@ -1541,7 +1874,7 @@ static int do_help(int argc, char **argv) " " HELP_SPEC_MAP "\n" " " HELP_SPEC_PROGRAM "\n" " " HELP_SPEC_OPTIONS " |\n" - " {-B|--base-btf} }\n" + " {[{-B|--base-btf} FILE]... }}\n" "", bin_name, "btf");
diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index e407582..3267163 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c
@@ -524,6 +524,15 @@ static int btf_dumper_var(const struct btf_dumper *d, __u32 type_id, return ret; } +/* + * The compiler splits a variable into pieces. Every piece is a VAR with the + * type of the whole variable, so there is less data than the type takes. + */ +bool btf_var_is_piece(const struct btf *btf, const struct btf_var_secinfo *vsi) +{ + return btf__resolve_size(btf, vsi->type) > vsi->size; +} + static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, const void *data) { @@ -542,6 +551,8 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, jsonw_name(d->jw, btf__name_by_offset(d->btf, t->name_off)); jsonw_start_array(d->jw); for (i = 0; i < vlen; i++) { + if (btf_var_is_piece(d->btf, &vsi[i])) + continue; ret = btf_dumper_do_type(d, vsi[i].type, 0, data + vsi[i].offset); if (ret) break;
diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c index ce69d1e..fee1a260 100644 --- a/tools/bpf/bpftool/cgroup.c +++ b/tools/bpf/bpftool/cgroup.c
@@ -56,7 +56,7 @@ static const int cgroup_attach_types[] = { }; #define HELP_SPEC_ATTACH_FLAGS \ - "ATTACH_FLAGS := { multi | override }" + "ATTACH_FLAGS := { [ multi | override ] [ preorder ] }" #define HELP_SPEC_ATTACH_TYPES \ " ATTACH_TYPE := { cgroup_inet_ingress | cgroup_inet_egress |\n" \ @@ -138,11 +138,69 @@ static void guess_vmlinux_btf_id(__u32 attach_btf_obj_id) close(fd); } +static const struct { + __u32 flag; + const char *name; +} attach_flag_names[] = { + { BPF_F_ALLOW_MULTI, "multi" }, + { BPF_F_ALLOW_OVERRIDE, "override" }, + { BPF_F_PREORDER, "preorder" }, +}; + +static const char *format_attach_flags(__u32 flags, char *buf, size_t sz) +{ + size_t len = 0; + size_t i; + int n; + + buf[0] = '\0'; + for (i = 0; i < ARRAY_SIZE(attach_flag_names); i++) { + if (flags & attach_flag_names[i].flag) { + n = snprintf(buf + len, sz - len, "%s%s", + len ? "," : "", attach_flag_names[i].name); + if (n < 0 || (size_t)n >= sz - len) + return buf; + len += n; + flags &= ~attach_flag_names[i].flag; + } + } + + if (flags) + snprintf(buf + len, sz - len, "%sunknown(%x)", + len ? "," : "", flags); + + return buf; +} + +static void show_attach_flags_json(__u32 flags) +{ + char buf[32]; + size_t i; + + jsonw_name(json_wtr, "attach_flags"); + jsonw_start_array(json_wtr); + + for (i = 0; i < ARRAY_SIZE(attach_flag_names); i++) { + if (!(flags & attach_flag_names[i].flag)) + continue; + + jsonw_string(json_wtr, attach_flag_names[i].name); + flags &= ~attach_flag_names[i].flag; + } + + if (flags) { + snprintf(buf, sizeof(buf), "unknown(%x)", flags); + jsonw_string(json_wtr, buf); + } + + jsonw_end_array(json_wtr); +} + static int show_bpf_prog(int id, enum bpf_attach_type attach_type, - const char *attach_flags_str, - int level) + __u32 attach_flags, int level) { char prog_name[MAX_PROG_FULL_NAME]; + char attach_flags_str[64]; const char *attach_btf_name = NULL; struct bpf_prog_info info = {}; const char *attach_type_str; @@ -182,7 +240,7 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, else jsonw_uint_field(json_wtr, "attach_type", attach_type); if (!(query_flags & BPF_F_QUERY_EFFECTIVE)) - jsonw_string_field(json_wtr, "attach_flags", attach_flags_str); + show_attach_flags_json(attach_flags); jsonw_string_field(json_wtr, "name", prog_name); if (attach_btf_name) jsonw_string_field(json_wtr, "attach_btf_name", attach_btf_name); @@ -198,7 +256,9 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, if (query_flags & BPF_F_QUERY_EFFECTIVE) printf(" %-15s", prog_name); else - printf(" %-15s %-15s", attach_flags_str, prog_name); + printf(" %-17s %-15s", + format_attach_flags(attach_flags, attach_flags_str, + sizeof(attach_flags_str)), prog_name); if (attach_btf_name) printf(" %-15s", attach_btf_name); else if (info.attach_btf_id) @@ -264,7 +324,7 @@ static int show_effective_bpf_progs(int cgroup_fd, enum bpf_attach_type type, return 0; for (iter = 0; iter < p.prog_cnt; iter++) - show_bpf_prog(prog_ids[iter], type, NULL, level); + show_bpf_prog(prog_ids[iter], type, 0, level); return 0; } @@ -274,9 +334,7 @@ static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, { LIBBPF_OPTS(bpf_prog_query_opts, p); __u32 prog_attach_flags[1024] = {0}; - const char *attach_flags_str; __u32 prog_ids[1024] = {0}; - char buf[32]; __u32 iter; int ret; @@ -296,24 +354,7 @@ static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, __u32 attach_flags; attach_flags = prog_attach_flags[iter] ?: p.attach_flags; - - switch (attach_flags) { - case BPF_F_ALLOW_MULTI: - attach_flags_str = "multi"; - break; - case BPF_F_ALLOW_OVERRIDE: - attach_flags_str = "override"; - break; - case 0: - attach_flags_str = ""; - break; - default: - snprintf(buf, sizeof(buf), "unknown(%x)", attach_flags); - attach_flags_str = buf; - } - - show_bpf_prog(prog_ids[iter], type, - attach_flags_str, level); + show_bpf_prog(prog_ids[iter], type, attach_flags, level); } return 0; @@ -377,7 +418,7 @@ static int do_show(int argc, char **argv) else if (query_flags & BPF_F_QUERY_EFFECTIVE) printf("%-8s %-15s %-15s\n", "ID", "AttachType", "Name"); else - printf("%-8s %-15s %-15s %-15s\n", "ID", "AttachType", + printf("%-8s %-15s %-17s %-15s\n", "ID", "AttachType", "AttachFlags", "Name"); btf_vmlinux = libbpf_find_kernel_btf(); @@ -531,7 +572,7 @@ static int do_show_tree(int argc, char **argv) "ID", "AttachType", "Name"); else printf("%s\n" - "%-8s %-15s %-15s %-15s\n", + "%-8s %-15s %-17s %-15s\n", "CgroupPath", "ID", "AttachType", "AttachFlags", "Name"); @@ -593,6 +634,8 @@ static int do_attach(int argc, char **argv) attach_flags |= BPF_F_ALLOW_MULTI; } else if (is_prefix(argv[i], "override")) { attach_flags |= BPF_F_ALLOW_OVERRIDE; + } else if (is_prefix(argv[i], "preorder")) { + attach_flags |= BPF_F_PREORDER; } else { p_err("unknown option: %s", argv[i]); goto exit_cgroup;
diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index ef366cc..ca5fa2f 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c
@@ -31,6 +31,7 @@ #include <bpf/bpf.h> #include <bpf/hashmap.h> #include <bpf/libbpf.h> /* libbpf_num_possible_cpus */ +#include <bpf/libbpf_internal.h> #include <bpf/btf.h> #include <zlib.h> @@ -71,6 +72,17 @@ void p_info(const char *fmt, ...) va_end(ap); } +/* + * Like strerror(), but also knows ENOTSUPP (524), which the kernel returns + * to user space from some BPF commands although it is not a userspace errno. + */ +const char *bpf_strerror(int err) +{ + if (err == 524 /* ENOTSUPP */) + return "Operation not supported"; + return strerror(err); +} + static bool is_bpffs(const char *path) { struct statfs st_fs; @@ -655,6 +667,45 @@ unsigned int get_possible_cpus(void) return cpus; } +int get_possible_cpu_ids(int **cpu_ids) +{ + const char *possible_cpus_file = "/sys/devices/system/cpu/possible"; + bool *mask = NULL; + int mask_sz, nr_cpus = 0; + int *ids = NULL; + int i, res; + + *cpu_ids = NULL; + + res = parse_cpu_mask_file(possible_cpus_file, &mask, &mask_sz); + if (res) { + p_err("failed to parse possible CPU mask: %s", strerror(-res)); + return res; + } + + for (i = 0; i < mask_sz; i++) + nr_cpus += mask[i]; + + ids = calloc(nr_cpus, sizeof(*ids)); + if (!ids) { + p_err("mem alloc failed"); + res = -ENOMEM; + goto out; + } + + for (i = 0, nr_cpus = 0; i < mask_sz; i++) { + if (mask[i]) + ids[nr_cpus++] = i; + } + *cpu_ids = ids; + ids = NULL; + res = nr_cpus; +out: + free(ids); + free(mask); + return res; +} + static char * ifindex_to_name_ns(__u32 ifindex, __u32 ns_dev, __u32 ns_ino, char *buf) {
diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index 0f6070a..846a58e 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c
@@ -394,9 +394,6 @@ static void probe_kernel_image_config(const char *define_prefix) /* xt_bpf module for passing BPF programs to netfilter */ { "CONFIG_NETFILTER_XT_MATCH_BPF", }, - /* test_bpf module for BPF tests */ - { "CONFIG_TEST_BPF", }, - /* Misc configs useful in BPF C programs */ /* jiffies <-> sec conversion for bpf_jiffies64() helper */ { "CONFIG_HZ", true, }
diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index a50540e..ee680f7 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c
@@ -500,16 +500,6 @@ static void print_hex(const char *data, int data_sz) } } -static size_t bpf_map_mmap_sz(const struct bpf_map *map) -{ - long page_sz = sysconf(_SC_PAGE_SIZE); - size_t map_sz; - - map_sz = (size_t)roundup(bpf_map__value_size(map), 8) * bpf_map__max_entries(map); - map_sz = roundup(map_sz, page_sz); - return map_sz; -} - /* Emit type size asserts for all top-level fields in memory-mapped internal maps. */ static void codegen_asserts(struct bpf_object *obj, const char *obj_name) { @@ -686,8 +676,8 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) if (!get_map_ident(map, ident, sizeof(ident))) continue; if (is_skel_data(map, ident, sizeof(ident))) - printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n", - ident, bpf_map_mmap_sz(map)); + printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$u, %3$u);\n", + ident, bpf_map__value_size(map), bpf_map__max_entries(map)); codegen("\ \n\ skel_closenz(skel->maps.%1$s.map_fd); \n\ @@ -771,13 +761,13 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h \n\ \"; \n\ \n\ - skel->%1$s = (__typeof__(skel->%1$s))skel_prep_map_data((void *)data, %2$zd,\n\ + skel->%1$s = (__typeof__(skel->%1$s))skel_prep_map_data((void *)data, %2$u, %3$u,\n\ sizeof(data) - 1);\n\ if (!skel->%1$s) \n\ goto cleanup; \n\ skel->maps.%1$s.initial_value = (__u64) (long) skel->%1$s;\n\ } \n\ - ", ident, bpf_map_mmap_sz(map)); + ", ident, bpf_map__value_size(map), bpf_map__max_entries(map)); } codegen("\ \n\ @@ -871,14 +861,14 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { codegen("\ \n\ - err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$zd);\n\ + err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$u, %3$u);\n\ if (err) \n\ return err; \n\ #ifdef __KERNEL__ \n\ skel->%1$s = NULL; \n\ #endif \n\ ", - ident, bpf_map_mmap_sz(map)); + ident, bpf_map__value_size(map), bpf_map__max_entries(map)); continue; } @@ -890,11 +880,11 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h codegen("\ \n\ skel->%1$s = (__typeof__(skel->%1$s))skel_finalize_map_data(&skel->maps.%1$s.initial_value,\n\ - %2$zd, %3$s, skel->maps.%1$s.map_fd);\n\ + %2$u, %3$u, %4$s, skel->maps.%1$s.map_fd);\n\ if (!skel->%1$s) \n\ return -ENOMEM; \n\ ", - ident, bpf_map_mmap_sz(map), mmap_flags); + ident, bpf_map__value_size(map), bpf_map__max_entries(map), mmap_flags); } codegen("\ \n\
diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 088d1d2..5cffa2d 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c
@@ -364,7 +364,7 @@ show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) } jsonw_uint_field(json_wtr, "cookie", data[j].cookie); jsonw_end_object(json_wtr); - if (j++ == info->kprobe_multi.count) + if (++j == info->kprobe_multi.count) break; } jsonw_end_array(json_wtr); @@ -867,7 +867,7 @@ static void show_kprobe_multi_plain(struct bpf_link_info *info) else printf(" "); - if (j++ == info->kprobe_multi.count) + if (++j == info->kprobe_multi.count) break; } error:
diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index c91e1a6..d15fec7 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c
@@ -31,6 +31,7 @@ bool block_mount; bool verifier_logs; bool relaxed_maps; bool use_loader; +bool wait_output; struct btf *base_btf; struct hashmap *refs_table; bool sign_progs; @@ -365,16 +366,20 @@ static int do_batch(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); - while (fgets(buf, sizeof(buf), fp)) { - cp = strchr(buf, '#'); - if (cp) - *cp = '\0'; + for (;;) { + errno = 0; + if (!fgets(buf, sizeof(buf), fp)) + break; if (strlen(buf) == sizeof(buf) - 1) { errno = E2BIG; break; } + cp = strchr(buf, '#'); + if (cp) + *cp = '\0'; + /* Append continuation lines if any (coming after a line ending * with '\' in the batch file). */ @@ -387,15 +392,15 @@ static int do_batch(int argc, char **argv) goto err_close; } - cp = strchr(contline, '#'); - if (cp) - *cp = '\0'; - if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) { p_err("command %u is too long", lines); err = -1; goto err_close; } + + cp = strchr(contline, '#'); + if (cp) + *cp = '\0'; buf[strlen(buf) - 2] = '\0'; strcat(buf, contline); } @@ -458,25 +463,17 @@ int main(int argc, char **argv) { "nomount", no_argument, NULL, 'n' }, { "debug", no_argument, NULL, 'd' }, { "use-loader", no_argument, NULL, 'L' }, + { "wait", no_argument, NULL, 'w' }, { "sign", no_argument, NULL, 'S' }, { "base-btf", required_argument, NULL, 'B' }, { 0 } }; + struct btf *new_base_btf = NULL, *root_base_btf = NULL; bool version_requested = false; int opt, ret; setlinebuf(stdout); -#ifdef USE_LIBCAP - /* Libcap < 2.63 hooks before main() to compute the number of - * capabilities of the running kernel, and doing so it calls prctl() - * which may fail and set errno to non-zero. - * Let's reset errno to make sure this does not interfere with the - * batch mode. - */ - errno = 0; -#endif - last_do_help = do_help; pretty_output = false; json_output = false; @@ -485,7 +482,7 @@ int main(int argc, char **argv) bin_name = "bpftool"; opterr = 0; - while ((opt = getopt_long(argc, argv, "VhpjfLmndSi:k:B:l", + while ((opt = getopt_long(argc, argv, "VhpjfLmndSi:k:B:lw", options, NULL)) >= 0) { switch (opt) { case 'V': @@ -521,16 +518,23 @@ int main(int argc, char **argv) verifier_logs = true; break; case 'B': - base_btf = btf__parse(optarg, NULL); - if (!base_btf) { + /* handle multi-split BTF */ + new_base_btf = btf__parse_split(optarg, base_btf); + if (!new_base_btf) { p_err("failed to parse base BTF at '%s': %d\n", optarg, -errno); return -1; } + base_btf = new_base_btf; + if (!root_base_btf) + root_base_btf = base_btf; break; case 'L': use_loader = true; break; + case 'w': + wait_output = true; + break; case 'S': sign_progs = true; use_loader = true; @@ -573,7 +577,20 @@ int main(int argc, char **argv) if (json_output) jsonw_destroy(&json_wtr); - btf__free(base_btf); + while (base_btf) { + bool is_root = base_btf == root_base_btf; + + new_base_btf = (struct btf *)btf__base_btf(base_btf); + btf__free(base_btf); + /* + * Do not free base BTF that is an owned .BTF.base ; leads + * to a double-free, so only free as far as the root base + * we explicitly read with -B above. + */ + if (is_root) + break; + base_btf = new_base_btf; + } return ret; }
diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 78b6e0e..b1eb4b5 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h
@@ -57,7 +57,7 @@ static inline void *u64_to_ptr(__u64 ptr) }) #define ERR_MAX_LEN 1024 -#define MAX_SIG_SIZE 4096 +#define MAX_SIG_SIZE 16384 #define BPF_TAG_FMT "%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx" @@ -89,6 +89,7 @@ extern bool block_mount; extern bool verifier_logs; extern bool relaxed_maps; extern bool use_loader; +extern bool wait_output; extern struct btf *base_btf; extern struct hashmap *refs_table; extern bool sign_progs; @@ -97,6 +98,7 @@ extern const char *cert_path; void __printf(1, 2) p_err(const char *fmt, ...); void __printf(1, 2) p_info(const char *fmt, ...); +const char *bpf_strerror(int err); bool is_prefix(const char *pfx, const char *str); int detect_common_prefix(const char *arg, ...); @@ -120,6 +122,7 @@ struct obj_refs { }; struct btf; +struct btf_var_secinfo; struct bpf_line_info; int build_pinned_obj_table(struct hashmap *table, @@ -216,6 +219,7 @@ void print_hex_data_json(uint8_t *data, size_t len); unsigned int get_page_size(void); unsigned int get_possible_cpus(void); +int get_possible_cpu_ids(int **cpu_ids); const char * ifindex_to_arch(__u32 ifindex, __u64 ns_dev, __u64 ns_ino, const char **opt); @@ -237,6 +241,7 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, const void *data); void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id, char *func_only, int size); +bool btf_var_is_piece(const struct btf *btf, const struct btf_var_secinfo *vsi); void btf_dump_linfo_plain(const struct btf *btf, const struct bpf_line_info *linfo,
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 684a8fb..4b65edf 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c
@@ -70,7 +70,7 @@ static void *alloc_value(struct bpf_map_info *info) static int do_dump_btf(const struct btf_dumper *d, struct bpf_map_info *map_info, void *key, - void *value) + void *value, const int *cpu_ids, int cpu_cnt) { __u32 value_id; int ret = 0; @@ -93,15 +93,15 @@ static int do_dump_btf(const struct btf_dumper *d, jsonw_name(d->jw, "value"); ret = btf_dumper_type(d, value_id, value); } else { - unsigned int i, n, step; + unsigned int step; + int i; jsonw_name(d->jw, "values"); jsonw_start_array(d->jw); - n = get_possible_cpus(); step = round_up(map_info->value_size, 8); - for (i = 0; i < n; i++) { + for (i = 0; i < cpu_cnt; i++) { jsonw_start_object(d->jw); - jsonw_int_field(d->jw, "cpu", i); + jsonw_int_field(d->jw, "cpu", cpu_ids[i]); jsonw_name(d->jw, "value"); ret = btf_dumper_type(d, value_id, value + i * step); jsonw_end_object(d->jw); @@ -130,7 +130,8 @@ static json_writer_t *get_btf_writer(void) } static void print_entry_json(struct bpf_map_info *info, unsigned char *key, - unsigned char *value, struct btf *btf) + unsigned char *value, struct btf *btf, + const int *cpu_ids, int cpu_cnt) { jsonw_start_object(json_wtr); @@ -150,12 +151,12 @@ static void print_entry_json(struct bpf_map_info *info, unsigned char *key, }; jsonw_name(json_wtr, "formatted"); - do_dump_btf(&d, info, key, value); + do_dump_btf(&d, info, key, value, cpu_ids, cpu_cnt); } } else { - unsigned int i, n, step; + unsigned int step; + int i; - n = get_possible_cpus(); step = round_up(info->value_size, 8); jsonw_name(json_wtr, "key"); @@ -163,10 +164,10 @@ static void print_entry_json(struct bpf_map_info *info, unsigned char *key, jsonw_name(json_wtr, "values"); jsonw_start_array(json_wtr); - for (i = 0; i < n; i++) { + for (i = 0; i < cpu_cnt; i++) { jsonw_start_object(json_wtr); - jsonw_int_field(json_wtr, "cpu", i); + jsonw_int_field(json_wtr, "cpu", cpu_ids[i]); jsonw_name(json_wtr, "value"); print_hex_data_json(value + i * step, @@ -183,7 +184,7 @@ static void print_entry_json(struct bpf_map_info *info, unsigned char *key, }; jsonw_name(json_wtr, "formatted"); - do_dump_btf(&d, info, key, value); + do_dump_btf(&d, info, key, value, cpu_ids, cpu_cnt); } } @@ -227,7 +228,7 @@ print_entry_error(struct bpf_map_info *map_info, void *key, int lookup_errno) print_hex_data_json(key, map_info->key_size); jsonw_name(json_wtr, "value"); jsonw_start_object(json_wtr); /* error */ - jsonw_string_field(json_wtr, "error", strerror(lookup_errno)); + jsonw_string_field(json_wtr, "error", bpf_strerror(lookup_errno)); jsonw_end_object(json_wtr); /* error */ jsonw_end_object(json_wtr); /* entry */ } else { @@ -240,12 +241,12 @@ print_entry_error(struct bpf_map_info *map_info, void *key, int lookup_errno) msg = "<cannot read>"; print_entry_error_msg(map_info, key, - msg ? : strerror(lookup_errno)); + msg ? : bpf_strerror(lookup_errno)); } } static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, - unsigned char *value) + unsigned char *value, const int *cpu_ids, int cpu_cnt) { if (!map_is_per_cpu(info->type)) { bool single_line, break_names; @@ -273,9 +274,9 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, printf("\n"); } else { - unsigned int i, n, step; + unsigned int step; + int i; - n = get_possible_cpus(); step = round_up(info->value_size, 8); if (info->key_size) { @@ -284,9 +285,9 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, printf("\n"); } if (info->value_size) { - for (i = 0; i < n; i++) { - printf("value (CPU %02u):%c", - i, info->value_size > 16 ? '\n' : ' '); + for (i = 0; i < cpu_cnt; i++) { + printf("value (CPU %02d):%c", + cpu_ids[i], info->value_size > 16 ? '\n' : ' '); fprint_hex(stdout, value + i * step, info->value_size, " "); printf("\n"); @@ -742,7 +743,7 @@ static int do_show(int argc, char **argv) static int dump_map_elem(int fd, void *key, void *value, struct bpf_map_info *map_info, struct btf *btf, - json_writer_t *btf_wtr) + json_writer_t *btf_wtr, const int *cpu_ids, int cpu_cnt) { if (bpf_map_lookup_elem(fd, key, value)) { print_entry_error(map_info, key, errno); @@ -750,7 +751,7 @@ static int dump_map_elem(int fd, void *key, void *value, } if (json_output) { - print_entry_json(map_info, key, value, btf); + print_entry_json(map_info, key, value, btf, cpu_ids, cpu_cnt); } else if (btf) { struct btf_dumper d = { .btf = btf, @@ -758,9 +759,9 @@ static int dump_map_elem(int fd, void *key, void *value, .is_plain_text = true, }; - do_dump_btf(&d, map_info, key, value); + do_dump_btf(&d, map_info, key, value, cpu_ids, cpu_cnt); } else { - print_entry_plain(map_info, key, value); + print_entry_plain(map_info, key, value, cpu_ids, cpu_cnt); } return 0; @@ -833,6 +834,8 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, void *key, *value, *prev_key; unsigned int num_elems = 0; struct btf *btf = NULL; + int *cpu_ids = NULL; + int cpu_cnt = 0; int err; key = malloc(info->key_size); @@ -845,6 +848,29 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, prev_key = NULL; + if (map_is_per_cpu(info->type)) { + cpu_cnt = get_possible_cpu_ids(&cpu_ids); + if (cpu_cnt < 0) { + err = cpu_cnt; + goto exit_free; + } + } + + /* Fail early to avoid an empty dump if map type cannot be iterated */ + if (bpf_map_get_next_key(fd, NULL, key) && errno != ENOENT) { + const char *map_type_str, *err_str; + + err_str = bpf_strerror(errno); + map_type_str = libbpf_bpf_map_type_str(info->type); + if (map_type_str) + p_err("can't dump %s map: %s", map_type_str, err_str); + else + p_err("can't dump map of type %u: %s", info->type, + err_str); + err = -1; + goto exit_free; + } + if (wtr) { err = get_map_kv_btf(info, &btf); if (err) { @@ -872,11 +898,15 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, while (true) { err = bpf_map_get_next_key(fd, prev_key, key); if (err) { - if (errno == ENOENT) + if (errno == ENOENT) { err = 0; + break; + } + p_err("can't get next key: %s", bpf_strerror(errno)); break; } - if (!dump_map_elem(fd, key, value, info, btf, wtr)) + if (!dump_map_elem(fd, key, value, info, btf, wtr, + cpu_ids, cpu_cnt)) num_elems++; prev_key = key; } @@ -885,7 +915,7 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, jsonw_end_array(wtr); /* elements */ if (show_header) jsonw_end_object(wtr); /* map object */ - } else { + } else if (!err) { printf("Found %u element%s\n", num_elems, num_elems != 1 ? "s" : ""); } @@ -893,6 +923,7 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); + free(cpu_ids); free_map_kv_btf(btf); return err; @@ -1019,7 +1050,7 @@ static int do_update(int argc, char **argv) err = bpf_map_update_elem(fd, key, value, flags); if (err) { - p_err("update failed: %s", strerror(errno)); + p_err("update failed: %s", bpf_strerror(errno)); goto exit_free; } @@ -1035,17 +1066,30 @@ static int do_update(int argc, char **argv) return err; } -static void print_key_value(struct bpf_map_info *info, void *key, - void *value) +static int print_key_value(struct bpf_map_info *info, void *key, + void *value) { json_writer_t *btf_wtr; struct btf *btf; + int *cpu_ids = NULL; + int cpu_cnt = 0; + int err = 0; - if (get_map_kv_btf(info, &btf)) - return; + if (map_is_per_cpu(info->type)) { + cpu_cnt = get_possible_cpu_ids(&cpu_ids); + + if (cpu_cnt < 0) { + err = cpu_cnt; + goto out; + } + } + + err = get_map_kv_btf(info, &btf); + if (err) + goto out; if (json_output) { - print_entry_json(info, key, value, btf); + print_entry_json(info, key, value, btf, cpu_ids, cpu_cnt); } else if (btf) { /* if here json_wtr wouldn't have been initialised, * so let's create separate writer for btf @@ -1055,7 +1099,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, p_info("failed to create json writer for btf. falling back to plain output"); free_map_kv_btf(btf); btf = NULL; - print_entry_plain(info, key, value); + print_entry_plain(info, key, value, cpu_ids, cpu_cnt); } else { struct btf_dumper d = { .btf = btf, @@ -1063,13 +1107,16 @@ static void print_key_value(struct bpf_map_info *info, void *key, .is_plain_text = true, }; - do_dump_btf(&d, info, key, value); + do_dump_btf(&d, info, key, value, cpu_ids, cpu_cnt); jsonw_destroy(&btf_wtr); } } else { - print_entry_plain(info, key, value); + print_entry_plain(info, key, value, cpu_ids, cpu_cnt); } free_map_kv_btf(btf); +out: + free(cpu_ids); + return err; } static int do_lookup(int argc, char **argv) @@ -1107,14 +1154,14 @@ static int do_lookup(int argc, char **argv) printf("\n\nNot found\n"); } } else { - p_err("lookup failed: %s", strerror(errno)); + p_err("lookup failed: %s", bpf_strerror(errno)); } goto exit_free; } /* here means bpf_map_lookup_elem() succeeded */ - print_key_value(&info, key, value); + err = print_key_value(&info, key, value); exit_free: free(key); @@ -1159,7 +1206,7 @@ static int do_getnext(int argc, char **argv) err = bpf_map_get_next_key(fd, key, nextkey); if (err) { - p_err("can't get next key: %s", strerror(errno)); + p_err("can't get next key: %s", bpf_strerror(errno)); goto exit_free; } @@ -1224,7 +1271,7 @@ static int do_delete(int argc, char **argv) err = bpf_map_delete_elem(fd, key); if (err) - p_err("delete failed: %s", strerror(errno)); + p_err("delete failed: %s", bpf_strerror(errno)); exit_free: free(key); @@ -1352,7 +1399,7 @@ static int do_create(int argc, char **argv) fd = bpf_map_create(map_type, map_name, key_size, value_size, max_entries, &attr); if (fd < 0) { - p_err("map create failed: %s", strerror(errno)); + p_err("map create failed: %s", bpf_strerror(errno)); goto exit; } @@ -1398,13 +1445,13 @@ static int do_pop_dequeue(int argc, char **argv) else printf("Error: empty map\n"); } else { - p_err("pop failed: %s", strerror(errno)); + p_err("pop failed: %s", bpf_strerror(errno)); } goto exit_free; } - print_key_value(&info, key, value); + err = print_key_value(&info, key, value); exit_free: free(key); @@ -1433,7 +1480,7 @@ static int do_freeze(int argc, char **argv) err = bpf_map_freeze(fd); close(fd); if (err) { - p_err("failed to freeze map: %s", strerror(errno)); + p_err("failed to freeze map: %s", bpf_strerror(errno)); return err; }
diff --git a/tools/bpf/bpftool/map_perf_ring.c b/tools/bpf/bpftool/map_perf_ring.c index bcb767e..5878938 100644 --- a/tools/bpf/bpftool/map_perf_ring.c +++ b/tools/bpf/bpftool/map_perf_ring.c
@@ -27,7 +27,7 @@ #define MMAP_PAGE_CNT 16 -static volatile bool stop; +static volatile sig_atomic_t stop; struct perf_event_sample { struct perf_event_header header; @@ -44,7 +44,6 @@ struct perf_event_lost { static void int_exit(int signo) { - fprintf(stderr, "Stopping...\n"); stop = true; } @@ -107,6 +106,27 @@ print_bpf_output(void *private_data, int cpu, struct perf_event_header *event) return LIBBPF_PERF_EVENT_CONT; } +static int print_ringbuf_output(void *ctx, void *data, size_t size) +{ + if (json_output) { + jsonw_start_object(json_wtr); + jsonw_uint_field(json_wtr, "size", size); + jsonw_name(json_wtr, "data"); + print_data_json(data, size); + jsonw_end_object(json_wtr); + } else { + printf("== size: %zu =====\n", size); + fprint_hex(stdout, data, size, " "); + printf("\n"); + } + + if (fflush(stdout)) + return errno ? -errno : -EIO; + + /* A producer can keep poll() busy even after a signal arrives. */ + return stop ? -EINTR : 0; +} + int do_event_pipe(int argc, char **argv) { struct perf_event_attr perf_attr = { @@ -123,18 +143,27 @@ int do_event_pipe(int argc, char **argv) .cpu = -1, .idx = -1, }; - struct perf_buffer *pb; + struct perf_buffer *pb = NULL; + struct ring_buffer *rb = NULL; __u32 map_info_len; int err, map_fd; + stop = false; map_info_len = sizeof(map_info); map_fd = map_parse_fd_and_info(&argc, &argv, &map_info, &map_info_len, 0); if (map_fd < 0) return -1; - if (map_info.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { - p_err("map is not a perf event array"); + if (map_info.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY && + map_info.type != BPF_MAP_TYPE_RINGBUF) { + p_err("map is not a perf event array or ring buffer"); + goto err_close_map; + } + + if (map_info.type == BPF_MAP_TYPE_RINGBUF && argc && + (is_prefix(*argv, "cpu") || is_prefix(*argv, "index"))) { + p_err("ring buffer maps do not support cpu or index arguments"); goto err_close_map; } @@ -184,15 +213,24 @@ int do_event_pipe(int argc, char **argv) ctx.idx = 0; } - opts.cpu_cnt = ctx.all_cpus ? 0 : 1; - opts.cpus = &ctx.cpu; - opts.map_keys = &ctx.idx; - pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr, - print_bpf_output, &ctx, &opts); - if (!pb) { - p_err("failed to create perf buffer: %s (%d)", - strerror(errno), errno); - goto err_close_map; + if (map_info.type == BPF_MAP_TYPE_RINGBUF) { + rb = ring_buffer__new(map_fd, print_ringbuf_output, NULL, NULL); + if (!rb) { + p_err("failed to create ring buffer: %s (%d)", + strerror(errno), errno); + goto err_close_map; + } + } else { + opts.cpu_cnt = ctx.all_cpus ? 0 : 1; + opts.cpus = &ctx.cpu; + opts.map_keys = &ctx.idx; + pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr, + print_bpf_output, &ctx, &opts); + if (!pb) { + p_err("failed to create perf buffer: %s (%d)", + strerror(errno), errno); + goto err_close_map; + } } signal(SIGINT, int_exit); @@ -202,25 +240,32 @@ int do_event_pipe(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); + err = 0; while (!stop) { - err = perf_buffer__poll(pb, 200); + err = rb ? ring_buffer__poll(rb, 200) : perf_buffer__poll(pb, 200); if (err < 0 && err != -EINTR) { - p_err("perf buffer polling failed: %s (%d)", - strerror(errno), errno); - goto err_close_pb; + p_err("%s buffer polling failed: %s (%d)", + rb ? "ring" : "perf", strerror(-err), -err); + break; } + err = 0; } + if (stop) + fprintf(stderr, "Stopping...\n"); if (json_output) jsonw_end_array(json_wtr); + if (fflush(stdout)) { + p_err("failed to write events: %s", strerror(errno)); + err = -1; + } + ring_buffer__free(rb); perf_buffer__free(pb); close(map_fd); - return 0; + return err < 0 ? -1 : 0; -err_close_pb: - perf_buffer__free(pb); err_close_map: close(map_fd); return -1;
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a9f730d..43599c7 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c
@@ -348,7 +348,7 @@ static void show_prog_metadata(int fd, __u32 num_maps) t_var = btf__type_by_id(btf, vsi->type); name = btf__name_by_offset(btf, t_var->name_off); - if (!has_metadata_prefix(name)) + if (!has_metadata_prefix(name) || btf_var_is_piece(btf, vsi)) continue; if (!printed_header) { @@ -377,7 +377,7 @@ static void show_prog_metadata(int fd, __u32 num_maps) t_var = btf__type_by_id(btf, vsi->type); name = btf__name_by_offset(btf, t_var->name_off); - if (!has_metadata_prefix(name)) + if (!has_metadata_prefix(name) || btf_var_is_piece(btf, vsi)) continue; if (!printed_header) { @@ -1119,21 +1119,66 @@ enum prog_tracelog_mode { TRACE_STDERR, }; +static void exit_stream(int signo) +{ + _exit(0); +} + +/* Consumes prog_fd. */ static int prog_tracelog_stream(int prog_fd, enum prog_tracelog_mode mode) { + const struct sigaction act = { .sa_handler = exit_stream }; + const int signals[] = { SIGHUP, SIGINT, SIGTERM }; FILE *file = mode == TRACE_STDOUT ? stdout : stderr; int stream_id = mode == TRACE_STDOUT ? 1 : 2; char buf[512]; - int ret; + unsigned int i; + int fd, ret; - ret = 0; - do { - ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL); - if (ret > 0) - fwrite(buf, sizeof(buf[0]), ret, file); - } while (ret > 0); + if (!wait_output) { + do { + ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL); + if (ret > 0) + fwrite(buf, sizeof(buf[0]), ret, file); + } while (ret > 0); + if (ret < 0) + p_err("failed to read stream: %s", strerror(-ret)); + close(prog_fd); + goto out; + } + fd = bpf_prog_stream_open(prog_fd, stream_id, NULL); + /* + * The stream descriptor does not keep the program alive. Drop the + * program reference so that reads return EOF once the program is gone. + */ + close(prog_fd); + if (fd == -EINVAL) { + /* The bpf() syscall rejects unknown commands with EINVAL. */ + p_err("waiting for stream output is not supported by this kernel"); + return -1; + } + if (fd < 0) { + p_err("failed to open stream: %s", strerror(-fd)); + return -1; + } + + /* + * Exit from the handler like the trace pipe variant does. A flag checked + * between reads would miss a signal that lands before read() blocks and + * only end the loop at the next print. + */ + for (i = 0; i < ARRAY_SIZE(signals); i++) + sigaction(signals[i], &act, NULL); + while ((ret = read(fd, buf, sizeof(buf))) > 0) { + fwrite(buf, sizeof(buf[0]), ret, file); + fflush(file); + } + if (ret < 0) + p_err("failed to read stream: %s", strerror(errno)); + close(fd); +out: fflush(file); return ret ? -1 : 0; } @@ -1454,7 +1499,7 @@ static int do_run(int argc, char **argv) err = bpf_prog_test_run_opts(fd, &test_attr); if (err) { - p_err("failed to run program: %s", strerror(errno)); + p_err("failed to run program: %s", bpf_strerror(errno)); goto free_ctx_out; } @@ -1769,7 +1814,7 @@ static int load_with_options(int argc, char **argv, bool first_prog_only) } if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) { - bpf_program__set_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY); bpf_program__set_ifindex(pos, xdpmeta_ifindex); } else { bpf_program__set_ifindex(pos, offload_ifindex); @@ -2062,37 +2107,52 @@ static int do_profile(int argc, char **argv) #include "profiler.skel.h" +enum ratio_metric { + METRIC_NONE = -2, + METRIC_RUN_CNT = -1, + METRIC_CYCLES = 0, + METRIC_INSTRUCTIONS = 1, + METRIC_L1D_LOADS = 2, + METRIC_LLC_MISSES = 3, + METRIC_ITLB_MISSES = 4, + METRIC_DTLB_MISSES = 5, +}; + struct profile_metric { const char *name; struct bpf_perf_event_value val; + __u64 scaled_val; struct perf_event_attr attr; bool selected; /* calculate ratios like instructions per cycle */ - const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */ + const enum ratio_metric ratio_metric; const char *ratio_desc; const float ratio_mul; } metrics[] = { - { + [METRIC_CYCLES] = { .name = "cycles", .attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES, .exclude_user = 1, }, + .ratio_metric = METRIC_RUN_CNT, + .ratio_desc = "cycles per run", + .ratio_mul = 1.0, }, - { + [METRIC_INSTRUCTIONS] = { .name = "instructions", .attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS, .exclude_user = 1, }, - .ratio_metric = 1, + .ratio_metric = METRIC_CYCLES, .ratio_desc = "insns per cycle", .ratio_mul = 1.0, }, - { + [METRIC_L1D_LOADS] = { .name = "l1d_loads", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2102,8 +2162,9 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), .exclude_user = 1, }, + .ratio_metric = METRIC_NONE, }, - { + [METRIC_LLC_MISSES] = { .name = "llc_misses", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2113,11 +2174,11 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), .exclude_user = 1 }, - .ratio_metric = 2, + .ratio_metric = METRIC_INSTRUCTIONS, .ratio_desc = "LLC misses per million insns", .ratio_mul = 1e6, }, - { + [METRIC_ITLB_MISSES] = { .name = "itlb_misses", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2127,11 +2188,11 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), .exclude_user = 1 }, - .ratio_metric = 2, + .ratio_metric = METRIC_INSTRUCTIONS, .ratio_desc = "itlb misses per million insns", .ratio_mul = 1e6, }, - { + [METRIC_DTLB_MISSES] = { .name = "dtlb_misses", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2141,13 +2202,16 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), .exclude_user = 1 }, - .ratio_metric = 2, + .ratio_metric = METRIC_INSTRUCTIONS, .ratio_desc = "dtlb misses per million insns", .ratio_mul = 1e6, }, }; static __u64 profile_total_count; +static int profile_cpu_cnt; +static int *profile_cpu_ids; +static int profile_cpu_id_span; #define MAX_NUM_PROFILE_METRICS 4 @@ -2182,9 +2246,9 @@ static int profile_parse_metrics(int argc, char **argv) return selected_cnt; } -static void profile_read_values(struct profiler_bpf *obj) +static int profile_read_values(struct profiler_bpf *obj) { - __u32 m, cpu, num_cpu = obj->rodata->num_cpu; + __u32 m, cpu, num_cpu = profile_cpu_cnt; int reading_map_fd, count_map_fd; __u64 counts[num_cpu]; __u32 key = 0; @@ -2192,15 +2256,11 @@ static void profile_read_values(struct profiler_bpf *obj) reading_map_fd = bpf_map__fd(obj->maps.accum_readings); count_map_fd = bpf_map__fd(obj->maps.counts); - if (reading_map_fd < 0 || count_map_fd < 0) { - p_err("failed to get fd for map"); - return; - } err = bpf_map_lookup_elem(count_map_fd, &key, counts); if (err) { p_err("failed to read count_map: %s", strerror(errno)); - return; + return err; } profile_total_count = 0; @@ -2208,24 +2268,37 @@ static void profile_read_values(struct profiler_bpf *obj) profile_total_count += counts[cpu]; for (m = 0; m < ARRAY_SIZE(metrics); m++) { - struct bpf_perf_event_value values[num_cpu]; + struct bpf_perf_event_value values[num_cpu], *val; + double scale; if (!metrics[m].selected) continue; err = bpf_map_lookup_elem(reading_map_fd, &key, values); if (err) { - p_err("failed to read reading_map: %s", - strerror(errno)); - return; + p_err("failed to read reading_map: %s", strerror(errno)); + return err; } + for (cpu = 0; cpu < num_cpu; cpu++) { - metrics[m].val.counter += values[cpu].counter; - metrics[m].val.enabled += values[cpu].enabled; - metrics[m].val.running += values[cpu].running; + val = &values[cpu]; + if (counts[cpu] && !val->running) { + p_err("perf event %s was not counted on CPU %d", + metrics[m].name, profile_cpu_ids[cpu]); + return -EAGAIN; + } + metrics[m].val.enabled += val->enabled; + metrics[m].val.running += val->running; + metrics[m].val.counter += val->counter; + if (val->running) { + /* Scale values to account for perf event multiplexing. */ + scale = (double)val->enabled / val->running; + metrics[m].scaled_val += val->counter * scale; + } } key++; } + return 0; } static void profile_print_readings_json(void) @@ -2242,6 +2315,7 @@ static void profile_print_readings_json(void) jsonw_lluint_field(json_wtr, "value", metrics[m].val.counter); jsonw_lluint_field(json_wtr, "enabled", metrics[m].val.enabled); jsonw_lluint_field(json_wtr, "running", metrics[m].val.running); + jsonw_lluint_field(json_wtr, "value_scaled", metrics[m].scaled_val); jsonw_end_object(json_wtr); } @@ -2250,24 +2324,34 @@ static void profile_print_readings_json(void) static void profile_print_readings_plain(void) { - __u32 m; + __u32 i; printf("\n%18llu %-20s\n", profile_total_count, "run_cnt"); - for (m = 0; m < ARRAY_SIZE(metrics); m++) { - struct bpf_perf_event_value *val = &metrics[m].val; + for (i = 0; i < ARRAY_SIZE(metrics); i++) { + struct profile_metric *m = &metrics[i]; + struct bpf_perf_event_value *val = &m->val; int r; + __u64 ratio; - if (!metrics[m].selected) + if (!m->selected) continue; - printf("%18llu %-20s", val->counter, metrics[m].name); + printf("%18llu %-20s", m->scaled_val, m->name); - r = metrics[m].ratio_metric - 1; - if (r >= 0 && metrics[r].selected && - metrics[r].val.counter > 0) { + r = m->ratio_metric; + switch (r) { + case METRIC_RUN_CNT: + ratio = profile_total_count; + break; + case METRIC_NONE: + ratio = 0; + break; + default: + ratio = metrics[r].scaled_val; + } + if (ratio) { printf("# %8.2f %-30s", - val->counter * metrics[m].ratio_mul / - metrics[r].val.counter, - metrics[m].ratio_desc); + m->scaled_val * m->ratio_mul / ratio, + m->ratio_desc); } else { printf("%-41s", ""); } @@ -2350,7 +2434,7 @@ static char *profile_tgt_name; static int *profile_perf_events; static int profile_perf_event_cnt; -static void profile_close_perf_events(struct profiler_bpf *obj) +static void profile_close_perf_events(void) { int i; @@ -2358,89 +2442,115 @@ static void profile_close_perf_events(struct profiler_bpf *obj) close(profile_perf_events[i]); free(profile_perf_events); + profile_perf_events = NULL; profile_perf_event_cnt = 0; } -static int profile_open_perf_event(int mid, int cpu, int map_fd) +static int profile_open_perf_event(int mid, int cpu, + int map_fd, int group_fd, __u32 map_key) { + struct perf_event_attr attr = metrics[mid].attr; + bool group_leader = group_fd < 0; int pmu_fd; + int err; - pmu_fd = syscall(__NR_perf_event_open, &metrics[mid].attr, - -1 /*pid*/, cpu, -1 /*group_fd*/, 0); + attr.disabled = group_leader; + pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, cpu, + group_fd, 0); if (pmu_fd < 0) { - if (errno == ENODEV) { - p_info("cpu %d may be offline, skip %s profiling.", - cpu, metrics[mid].name); - profile_perf_event_cnt++; - return 0; - } - return -1; + err = -errno; + if (errno == ENODEV && group_leader) + p_info("cpu %d may be offline, skip profiling.", cpu); + return err; } - if (bpf_map_update_elem(map_fd, - &profile_perf_event_cnt, - &pmu_fd, BPF_ANY) || - ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) { + if (bpf_map_update_elem(map_fd, &map_key, &pmu_fd, BPF_ANY)) { + err = -errno; close(pmu_fd); - return -1; + return err; } profile_perf_events[profile_perf_event_cnt++] = pmu_fd; - return 0; + return pmu_fd; } static int profile_open_perf_events(struct profiler_bpf *obj) { + __u32 map_key; unsigned int cpu, m; - int map_fd; + int group_fd, map_fd, pmu_fd; + int err; - profile_perf_events = calloc( - obj->rodata->num_cpu * obj->rodata->num_metric, sizeof(int)); + profile_perf_events = calloc(profile_cpu_cnt * obj->rodata->num_metric, + sizeof(*profile_perf_events)); if (!profile_perf_events) { p_err("failed to allocate memory for perf_event array: %s", strerror(errno)); - return -1; - } - map_fd = bpf_map__fd(obj->maps.events); - if (map_fd < 0) { - p_err("failed to get fd for events map"); - return -1; + return -ENOMEM; } - for (m = 0; m < ARRAY_SIZE(metrics); m++) { - if (!metrics[m].selected) - continue; - for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) { - if (profile_open_perf_event(m, cpu, map_fd)) { - p_err("failed to create event %s on cpu %u", - metrics[m].name, cpu); - return -1; + map_fd = bpf_map__fd(obj->maps.events); + for (cpu = 0; cpu < (unsigned int)profile_cpu_cnt; cpu++) { + int cpu_id = profile_cpu_ids[cpu]; + + group_fd = -1; + map_key = cpu_id; + for (m = 0; m < ARRAY_SIZE(metrics); m++) { + if (!metrics[m].selected) + continue; + + pmu_fd = profile_open_perf_event(m, cpu_id, map_fd, group_fd, + map_key); + if (pmu_fd == -ENODEV && group_fd < 0) + break; + if (pmu_fd < 0) { + p_err("failed to add event %s to group on CPU %d: %s", + metrics[m].name, cpu_id, strerror(-pmu_fd)); + return pmu_fd; } + if (group_fd < 0) + group_fd = pmu_fd; + map_key += profile_cpu_id_span; + } + if (group_fd < 0) + continue; + if (ioctl(group_fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP)) { + err = -errno; + p_err("failed to enable perf event group on CPU %d: %s", + cpu_id, strerror(-err)); + return err; } } return 0; } -static void profile_print_and_cleanup(void) +static int profile_print_and_cleanup(void) { - profile_close_perf_events(profile_obj); - profile_read_values(profile_obj); - profile_print_readings(); + int err; + + profile_close_perf_events(); + err = profile_read_values(profile_obj); + if (!err) + profile_print_readings(); profiler_bpf__destroy(profile_obj); close(profile_tgt_fd); free(profile_tgt_name); + free(profile_cpu_ids); + profile_cpu_ids = NULL; + profile_cpu_cnt = 0; + profile_cpu_id_span = 0; + return err; } static void int_exit(int signo) { - profile_print_and_cleanup(); - exit(0); + exit(!!profile_print_and_cleanup()); } static int do_profile(int argc, char **argv) { - int num_metric, num_cpu, err = -1; + int num_metric, err = -1; struct bpf_program *prog; unsigned long duration; char *endptr; @@ -2471,11 +2581,12 @@ static int do_profile(int argc, char **argv) if (num_metric <= 0) goto out; - num_cpu = libbpf_num_possible_cpus(); - if (num_cpu <= 0) { + profile_cpu_cnt = get_possible_cpu_ids(&profile_cpu_ids); + if (profile_cpu_cnt <= 0) { p_err("failed to identify number of CPUs"); goto out; } + profile_cpu_id_span = profile_cpu_ids[profile_cpu_cnt - 1] + 1; profile_obj = profiler_bpf__open(); if (!profile_obj) { @@ -2483,11 +2594,12 @@ static int do_profile(int argc, char **argv) goto out; } - profile_obj->rodata->num_cpu = num_cpu; profile_obj->rodata->num_metric = num_metric; + profile_obj->rodata->cpu_id_span = profile_cpu_id_span; /* adjust map sizes */ - bpf_map__set_max_entries(profile_obj->maps.events, num_metric * num_cpu); + bpf_map__set_max_entries(profile_obj->maps.events, + num_metric * profile_cpu_id_span); bpf_map__set_max_entries(profile_obj->maps.fentry_readings, num_metric); bpf_map__set_max_entries(profile_obj->maps.accum_readings, num_metric); bpf_map__set_max_entries(profile_obj->maps.counts, 1); @@ -2525,15 +2637,18 @@ static int do_profile(int argc, char **argv) signal(SIGINT, int_exit); sleep(duration); - profile_print_and_cleanup(); - return 0; + return profile_print_and_cleanup(); out: - profile_close_perf_events(profile_obj); + profile_close_perf_events(); if (profile_obj) profiler_bpf__destroy(profile_obj); close(profile_tgt_fd); free(profile_tgt_name); + free(profile_cpu_ids); + profile_cpu_ids = NULL; + profile_cpu_cnt = 0; + profile_cpu_id_span = 0; return err; } @@ -2589,7 +2704,8 @@ static int do_help(int argc, char **argv) " METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n" " " HELP_SPEC_OPTIONS " |\n" " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} |\n" - " {-L|--use-loader} | [ {-S|--sign } {-k} <private_key.pem> {-i} <certificate.x509> ] \n" + " {-L|--use-loader} | [ {-S|--sign } {-k} <private_key.pem> {-i} <certificate.x509> ] |\n" + " {-w|--wait} }\n" "", bin_name, argv[-2]);
diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index 88726a6..adbfd4a 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c
@@ -130,6 +130,12 @@ __u32 register_session_key(const char *key_der_path) int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) { + unsigned int signer_flags = CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP | +#ifdef CMS_NO_SIGNING_TIME + CMS_NO_SIGNING_TIME | +#endif + CMS_USE_KEYID | CMS_NOATTR; + const EVP_MD *cms_digest = EVP_sha256(); BIO *bd_in = NULL, *bd_out = NULL; EVP_PKEY *private_key = NULL; CMS_ContentInfo *cms = NULL; @@ -167,6 +173,20 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L + if (EVP_PKEY_is_a(private_key, "ML-DSA-44") || + EVP_PKEY_is_a(private_key, "ML-DSA-65") || + EVP_PKEY_is_a(private_key, "ML-DSA-87")) { + /* + * See workaround in 0ad9a71933e7 ("modsign: Enable ML-DSA + * module signing"). Kernel only accepts sha512, see also + * 8bbdeb7a25b4 ("pkcs7, x509: Add ML-DSA support"). + */ + signer_flags &= ~CMS_NOATTR; + cms_digest = EVP_sha512(); + } +#endif + cms = CMS_sign(NULL, NULL, NULL, NULL, CMS_NOCERTS | CMS_PARTIAL | CMS_BINARY | CMS_DETACHED | CMS_STREAM); @@ -175,9 +195,7 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - if (!CMS_add1_signer(cms, x509, private_key, EVP_sha256(), - CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP | - CMS_USE_KEYID | CMS_NOATTR)) { + if (!CMS_add1_signer(cms, x509, private_key, cms_digest, signer_flags)) { err = -EINVAL; goto cleanup; }
diff --git a/tools/bpf/bpftool/skeleton/profiler.bpf.c b/tools/bpf/bpftool/skeleton/profiler.bpf.c index f48c783..0d47a9b 100644 --- a/tools/bpf/bpftool/skeleton/profiler.bpf.c +++ b/tools/bpf/bpftool/skeleton/profiler.bpf.c
@@ -10,7 +10,7 @@ struct bpf_perf_event_value___local { __u64 running; } __attribute__((preserve_access_index)); -/* map of perf event fds, num_cpu * num_metric entries */ +/* map of perf event fds, cpu_id_span * num_metric entries */ struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(u32)); @@ -38,36 +38,29 @@ struct { __uint(value_size, sizeof(u64)); } counts SEC(".maps"); -const volatile __u32 num_cpu = 1; const volatile __u32 num_metric = 1; +const volatile __u32 cpu_id_span = 1; #define MAX_NUM_METRICS 4 SEC("fentry/XXX") int BPF_PROG(fentry_XXX) { - struct bpf_perf_event_value___local *ptrs[MAX_NUM_METRICS]; u32 key = bpf_get_smp_processor_id(); u32 i; - /* look up before reading, to reduce error */ for (i = 0; i < num_metric && i < MAX_NUM_METRICS; i++) { + struct bpf_perf_event_value___local *reading; u32 flag = i; - - ptrs[i] = bpf_map_lookup_elem(&fentry_readings, &flag); - if (!ptrs[i]) - return 0; - } - - for (i = 0; i < num_metric && i < MAX_NUM_METRICS; i++) { - struct bpf_perf_event_value___local reading; int err; - err = bpf_perf_event_read_value(&events, key, (void *)&reading, - sizeof(reading)); + reading = bpf_map_lookup_elem(&fentry_readings, &flag); + if (!reading) + return 0; + err = bpf_perf_event_read_value(&events, key, (void *)reading, + sizeof(*reading)); if (err) return 0; - *(ptrs[i]) = reading; - key += num_cpu; + key += cpu_id_span; } return 0; @@ -80,7 +73,7 @@ fexit_update_maps(u32 id, struct bpf_perf_event_value___local *after) before = bpf_map_lookup_elem(&fentry_readings, &id); /* only account samples with a valid fentry_reading */ - if (before && before->counter) { + if (before && before->enabled) { struct bpf_perf_event_value___local *accum; diff.counter = after->counter - before->counter; @@ -107,7 +100,7 @@ int BPF_PROG(fexit_XXX) /* read all events before updating the maps, to reduce error */ for (i = 0; i < num_metric && i < MAX_NUM_METRICS; i++) { - err = bpf_perf_event_read_value(&events, cpu + i * num_cpu, + err = bpf_perf_event_read_value(&events, cpu + i * cpu_id_span, (void *)(readings + i), sizeof(*readings)); if (err)
diff --git a/tools/bpf/bpftool/tracelog.c b/tools/bpf/bpftool/tracelog.c index 573a8d9..e104748 100644 --- a/tools/bpf/bpftool/tracelog.c +++ b/tools/bpf/bpftool/tracelog.c
@@ -86,7 +86,7 @@ static bool get_tracefs_pipe(char *mnt) fclose(fp); /* The string from fscanf() might be truncated, check mnt is valid */ - if (found && validate_tracefs_mnt(mnt, TRACEFS_MAGIC)) + if (found && !validate_tracefs_mnt(mnt, TRACEFS_MAGIC)) goto exit_found; if (block_mount)
diff --git a/tools/bpf/resolve_btfids/Build b/tools/bpf/resolve_btfids/Build index 077de38..602e150 100644 --- a/tools/bpf/resolve_btfids/Build +++ b/tools/bpf/resolve_btfids/Build
@@ -1,6 +1,7 @@ hostprogs := resolve_btfids resolve_btfids-y += main.o +resolve_btfids-y += btf_colors.o resolve_btfids-y += rbtree.o resolve_btfids-y += zalloc.o resolve_btfids-y += string.o
diff --git a/tools/bpf/resolve_btfids/btf_colors.c b/tools/bpf/resolve_btfids/btf_colors.c new file mode 100644 index 0000000..5d307d7 --- /dev/null +++ b/tools/bpf/resolve_btfids/btf_colors.c
@@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <bpf/libbpf_internal.h> +#include "btf_colors.h" + +/* + * Functions in this file mainly exist to refer to functions from libbpf_internal.h, + * which can't be included in main.c because of the u32 poison and pr_warn macro conflicts. + */ + +void btf_sha256(const void *data, size_t len, + __u8 out[BTF_SHA256_DIGEST_LENGTH]) +{ + libbpf_sha256(data, len, out); +} + +/* + * Marks `root_id` and local types reachable from it with `color`. + * `colors` is indexed by source ID; worklist has room for every local type. + */ +void btf_mark_reachable(struct btf *btf, __u32 root_id, enum btf_color color, + __u8 *colors, __u32 *worklist) +{ + const struct btf *base = btf__base_btf(btf); + __u32 start_id = base ? btf__type_cnt(base) : 1; + __u32 pending = 0; + + if (root_id < start_id || (colors[root_id] & color) == color) + return; + colors[root_id] |= color; + worklist[pending++] = root_id; + while (pending) { + const struct btf_type *t = btf__type_by_id(btf, worklist[--pending]); + struct btf_field_iter it; + __u32 *id; + + btf_field_iter_init(&it, (struct btf_type *)t, BTF_FIELD_ITER_IDS); + while ((id = btf_field_iter_next(&it))) { + if (*id < start_id || (colors[*id] & color) == color) + continue; + colors[*id] |= color; + worklist[pending++] = *id; + } + } +} + +static int cmp_loc(const void *a, const void *b) +{ + const struct btf_loc *la = a, *lb = b; + + if (la->func != lb->func) + return la->func < lb->func ? -1 : 1; + if (la->offset != lb->offset) + return la->offset < lb->offset ? -1 : 1; + if (la->loc_proto != lb->loc_proto) + return la->loc_proto < lb->loc_proto ? -1 : 1; + return 0; +} + +static void remap(struct btf *btf, const __u32 *id_map, __u32 src_start_id) +{ + const struct btf *base = btf__base_btf(btf); + __u32 start_id = base ? btf__type_cnt(base) : 1; + __u32 i, type_cnt = btf__type_cnt(btf); + + for (i = start_id; i < type_cnt; i++) { + struct btf_type *t = (struct btf_type *)btf__type_by_id(btf, i); + struct btf_field_iter it; + __u32 *id; + + btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); + while ((id = btf_field_iter_next(&it))) { + /* Void and types in the original ancestor are unchanged. */ + if (*id >= src_start_id) + *id = id_map[*id - src_start_id]; + } + if (btf_is_locsec(t)) + qsort(btf_locsec_locs(t), btf_vlen(t), sizeof(struct btf_loc), cmp_loc); + } +} + +/* + * Split the `src` into `main_out` base and `inline_out`, + * according to `colors` array. Relative ordering remains + * the same as in `src`. + */ +int btf_split_by_color(struct btf *src, const __u8 *colors, + struct btf **main_out, struct btf **inline_out) +{ + LIBBPF_OPTS(btf_new_opts, opts, + .base_btf = (struct btf *)btf__base_btf(src), + .add_layout = btf_header(src)->layout_len != 0, + ); + struct btf *main_btf, *inline_btf = NULL; + __u32 start_id = opts.base_btf ? btf__type_cnt(opts.base_btf) : 1; + __u32 type_cnt = btf__type_cnt(src); + __u32 *id_map, i; + int err = -ENOMEM; + + *main_out = NULL; + *inline_out = NULL; + id_map = malloc((type_cnt - start_id ?: 1) * sizeof(*id_map)); + if (!id_map) + return -ENOMEM; + main_btf = btf__new_empty_opts(&opts); + if (!main_btf) + goto out; + btf__set_endianness(main_btf, btf__endianness(src)); + /* Copy MAIN and SHARED marked types to `main_out`. */ + for (i = start_id; i < type_cnt; i++) { + if (colors[i] == BTF_COLOR_LOC) + continue; + err = btf__add_type(main_btf, src, btf__type_by_id(src, i)); + if (err < 0) + goto out; + id_map[i - start_id] = err; + } + /* + * Copy LOC marked types to `inline_out`. + * `main_btf` types count is stable at this point. + */ + inline_btf = btf__new_empty_split(main_btf); + if (!inline_btf) { + err = -ENOMEM; + goto out; + } + for (i = start_id; i < type_cnt; i++) { + if (colors[i] != BTF_COLOR_LOC) + continue; + err = btf__add_type(inline_btf, src, btf__type_by_id(src, i)); + if (err < 0) + goto out; + id_map[i - start_id] = err; + } + remap(main_btf, id_map, start_id); + remap(inline_btf, id_map, start_id); + *main_out = main_btf; + *inline_out = inline_btf; + free(id_map); + return 0; +out: + btf__free(inline_btf); + btf__free(main_btf); + free(id_map); + return err; +}
diff --git a/tools/bpf/resolve_btfids/btf_colors.h b/tools/bpf/resolve_btfids/btf_colors.h new file mode 100644 index 0000000..56a85a2 --- /dev/null +++ b/tools/bpf/resolve_btfids/btf_colors.h
@@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __RESOLVE_BTFIDS_BTF_COLORS_H +#define __RESOLVE_BTFIDS_BTF_COLORS_H + +#include <bpf/btf.h> + +enum btf_color { + BTF_COLOR_NONE = 0, + BTF_COLOR_MAIN = 1, + BTF_COLOR_LOC = 2, + BTF_COLOR_SHARED = BTF_COLOR_MAIN | BTF_COLOR_LOC, +}; + +#define BTF_SHA256_DIGEST_LENGTH 32 + +void btf_sha256(const void *data, size_t len, + __u8 out[BTF_SHA256_DIGEST_LENGTH]); +void btf_mark_reachable(struct btf *btf, __u32 root, enum btf_color color, + __u8 *colors, __u32 *worklist); + +int btf_split_by_color(struct btf *src, const __u8 *colors, + struct btf **main_out, struct btf **inline_out); + +#endif
diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 37d7e72..5ad5473 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c
@@ -89,9 +89,11 @@ #include <linux/limits.h> #include <bpf/btf.h> #include <bpf/libbpf.h> +#include "btf_colors.h" #include <subcmd/parse-options.h> #define BTF_IDS_SECTION ".BTF_ids" +#define BTF_LINK_MODULE_NAME_MAX 64 #define BTF_ID_PREFIX "__BTF_ID__" #define BTF_STRUCT "struct" @@ -142,7 +144,10 @@ struct object { struct btf *btf; struct btf *base_btf; + struct btf *inline_btf; + struct btf *orig_btf; bool distill_base; + bool extract_inline; struct { int fd; @@ -154,6 +159,11 @@ struct object { size_t strtabidx; unsigned long idlist_addr; int encoding; + int elf_class; + const char **func_symbols; + u32 func_symbols_cnt; + u32 func_symbols_cap; + bool func_symbols_collected; } efile; struct rb_root sets; @@ -172,6 +182,19 @@ struct object { u32 addr_syms_cap; }; +struct btf_link { + char *value; + char *section; + char *module; + char *btf_path; +}; + +struct btf_links { + struct btf_link *links; + u32 cnt; + u32 cap; +}; + #define DECL_TAG_FASTCALL "bpf_fastcall" #define DECL_TAG_KFUNC "bpf_kfunc" @@ -259,6 +282,49 @@ static int __ensure_mem(void **data, u32 *cap, u32 cnt, size_t elem_sz) #define ensure_mem(arr_ptr, cap_ptr, cnt) \ __ensure_mem((void **)(arr_ptr), (cap_ptr), (cnt), sizeof(**(arr_ptr))) +static int parse_btf_link(const struct option *opt, const char *arg, int unset) +{ + struct btf_links *links = opt->value; + struct btf_link *link; + char *separator; + + if (unset) + return -EINVAL; + if (__ensure_mem((void **)&links->links, &links->cap, links->cnt + 1, + sizeof(*links->links))) + return -ENOMEM; + link = &links->links[links->cnt]; + memset(link, 0, sizeof(*link)); + link->value = strdup(arg); + if (!link->value) + return -ENOMEM; + link->section = link->value; + separator = strchr(link->section, ':'); + if (!separator || separator == link->section) + goto err_value; + *separator++ = '\0'; + link->module = separator; + separator = strchr(link->module, ':'); + if (!separator || separator == link->module || !separator[1]) + goto err_value; + *separator++ = '\0'; + link->btf_path = separator; + links->cnt++; + return 0; +err_value: + free(link->value); + return -EINVAL; +} + +static void free_btf_links(struct btf_links *links) +{ + u32 i; + + for (i = 0; i < links->cnt; i++) + free(links->links[i].value); + free(links->links); +} + static bool is_btf_id(const char *name) { return name && !strncmp(name, BTF_ID_PREFIX, sizeof(BTF_ID_PREFIX) - 1); @@ -480,6 +546,7 @@ static int elf_collect(struct object *obj) return -1; } obj->efile.encoding = ehdr.e_ident[EI_DATA]; + obj->efile.elf_class = ehdr.e_ident[EI_CLASS]; /* * Scan all the elf sections and look for save data @@ -572,6 +639,58 @@ static const char *find_name_by_addr(struct object *obj, Elf64_Addr addr) return res ? res->name : NULL; } +static int cmp_func_symbol(const void *a, const void *b) +{ + const char * const *name = a; + const char * const *other = b; + + return strcmp(*name, *other); +} + +static int collect_func_symbols(struct object *obj) +{ + Elf_Scn *scn; + GElf_Shdr sh; + int n, i; + + if (obj->efile.symbols_shndx == -1) + return 0; + + scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx); + if (!scn || gelf_getshdr(scn, &sh) != &sh || !sh.sh_entsize) + return -EINVAL; + n = sh.sh_size / sh.sh_entsize; + + for (i = 0; i < n; i++) { + GElf_Sym sym; + const char *name; + + if (!gelf_getsym(obj->efile.symbols, i, &sym)) + return -EINVAL; + if (GELF_ST_TYPE(sym.st_info) != STT_FUNC || + sym.st_shndx == SHN_UNDEF || !sym.st_name) + continue; + name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, sym.st_name); + if (!name) + return -EINVAL; + if (ensure_mem(&obj->efile.func_symbols, &obj->efile.func_symbols_cap, + obj->efile.func_symbols_cnt + 1)) + return -ENOMEM; + obj->efile.func_symbols[obj->efile.func_symbols_cnt++] = name; + } + + qsort(obj->efile.func_symbols, obj->efile.func_symbols_cnt, + sizeof(*obj->efile.func_symbols), cmp_func_symbol); + obj->efile.func_symbols_collected = true; + return 0; +} + +static bool has_func_symbol(const struct object *obj, const char *name) +{ + return bsearch(&name, obj->efile.func_symbols, obj->efile.func_symbols_cnt, + sizeof(*obj->efile.func_symbols), cmp_func_symbol) != NULL; +} + static int symbols_collect(struct object *obj) { Elf_Scn *scn = NULL; @@ -688,8 +807,9 @@ static int load_btf(struct object *obj) if (err) { pr_err("FAILED: load base BTF from %s: %s\n", obj->base_btf_path, strerror(-err)); - goto out_err; + return err; } + obj->base_btf = base_btf; } btf = btf__parse_split(obj->btf_path ?: obj->path, base_btf); @@ -697,20 +817,12 @@ static int load_btf(struct object *obj) if (err) { pr_err("FAILED: load BTF from %s: %s\n", obj->btf_path ?: obj->path, strerror(-err)); - goto out_err; + return err; } - obj->base_btf = base_btf; obj->btf = btf; return 0; - -out_err: - btf__free(base_btf); - btf__free(btf); - obj->base_btf = NULL; - obj->btf = NULL; - return err; } static int symbols_resolve(struct object *obj) @@ -977,6 +1089,153 @@ static int dump_raw_btf(struct btf *btf, const char *out_path) return 0; } +static int patch_btf_link(struct object *obj, const struct btf_link *link) +{ + char section[128]; + void *raw_btf_data = NULL; + size_t shdrstrndx, module_name_len; + int fd = -1, len, err = -1; + FILE *btf_file = NULL; + Elf_Scn *scn = NULL; + u32 raw_btf_size; + struct stat st; + Elf_Data *data; + GElf_Shdr sh; + char *name; + Elf *elf = NULL; + u8 *size; + + if (stat(link->btf_path, &st)) { + err = -errno; + pr_err("FAILED to stat BTF file %s: %s\n", + link->btf_path, strerror(errno)); + return err; + } + if (!S_ISREG(st.st_mode) || st.st_size <= 0 || st.st_size > UINT_MAX) { + pr_err("FAILED: invalid BTF file size or type: %s\n", link->btf_path); + return -EINVAL; + } + raw_btf_size = st.st_size; + raw_btf_data = malloc(raw_btf_size); + if (!raw_btf_data) { + pr_err("FAILED to allocate %u bytes for BTF file %s\n", + raw_btf_size, link->btf_path); + return -ENOMEM; + } + btf_file = fopen(link->btf_path, "rb"); + if (!btf_file) { + err = -errno; + pr_err("FAILED to open BTF file %s: %s\n", + link->btf_path, strerror(errno)); + goto out; + } + if (fread(raw_btf_data, raw_btf_size, 1, btf_file) != 1) { + err = -EIO; + pr_err("FAILED to read BTF file %s\n", link->btf_path); + goto out; + } + fclose(btf_file); + btf_file = NULL; + if (obj->efile.elf_class == ELFCLASS32) { + module_name_len = BTF_LINK_MODULE_NAME_MAX - 4; + } else if (obj->efile.elf_class == ELFCLASS64) { + module_name_len = BTF_LINK_MODULE_NAME_MAX - 8; + } else { + err = -EINVAL; + pr_err("FAILED: unsupported ELF class %u in %s\n", + obj->efile.elf_class, obj->path); + goto out; + } + if (strlen(link->module) >= module_name_len) { + err = -EINVAL; + pr_err("FAILED: module name too long for BTF link: %s\n", link->module); + goto out; + } + len = snprintf(section, sizeof(section), "%s.link", link->section); + if (len < 0 || len >= sizeof(section)) { + err = -ENAMETOOLONG; + pr_err("FAILED: BTF link section name too long: %s.link\n", link->section); + goto out; + } + + fd = open(obj->path, O_RDWR); + if (fd < 0) { + err = -errno; + pr_err("FAILED to open %s: %s\n", obj->path, strerror(errno)); + goto out; + } + elf = elf_begin(fd, ELF_C_RDWR_MMAP, NULL); + if (!elf) { + err = -EINVAL; + pr_err("FAILED to create ELF descriptor for %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + elf_flagelf(elf, ELF_C_SET, ELF_F_LAYOUT); + if (elf_getshdrstrndx(elf, &shdrstrndx)) { + pr_err("FAILED to get section name index from %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + while ((scn = elf_nextscn(elf, scn))) { + if (gelf_getshdr(scn, &sh) != &sh) { + pr_err("FAILED to get section header from %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + name = elf_strptr(elf, shdrstrndx, sh.sh_name); + if (name && !strcmp(name, section)) + break; + } + if (!scn) { + pr_err("FAILED: section %s not found in %s\n", section, obj->path); + goto out; + } + data = elf_getdata(scn, NULL); + if (sh.sh_type != SHT_PROGBITS || !data || !data->d_buf || + data->d_size != module_name_len + + BTF_SHA256_DIGEST_LENGTH + sizeof(u32)) { + pr_err("FAILED: section %s is not a writable BTF link\n", section); + goto out; + } + memset(data->d_buf, 0, data->d_size); + memcpy(data->d_buf, link->module, strlen(link->module) + 1); + btf_sha256(raw_btf_data, raw_btf_size, + (u8 *)data->d_buf + module_name_len); + size = (u8 *)data->d_buf + module_name_len + BTF_SHA256_DIGEST_LENGTH; + if (obj->efile.encoding == ELFDATA2LSB) { + size[0] = raw_btf_size; + size[1] = raw_btf_size >> 8; + size[2] = raw_btf_size >> 16; + size[3] = raw_btf_size >> 24; + } else if (obj->efile.encoding == ELFDATA2MSB) { + size[0] = raw_btf_size >> 24; + size[1] = raw_btf_size >> 16; + size[2] = raw_btf_size >> 8; + size[3] = raw_btf_size; + } else { + pr_err("FAILED: unsupported ELF data encoding %u in %s\n", + obj->efile.encoding, obj->path); + goto out; + } + elf_flagdata(data, ELF_C_SET, ELF_F_DIRTY); + if (elf_update(elf, ELF_C_WRITE) < 0) { + pr_err("FAILED to update ELF file %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + err = 0; +out: + if (elf) + elf_end(elf); + if (fd >= 0) + close(fd); + if (btf_file) + fclose(btf_file); + free(raw_btf_data); + return err; +} + static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, s32 type_id) { const struct btf_type *t = btf__type_by_id(btf, type_id); @@ -1510,6 +1769,84 @@ static int btf2btf(struct object *obj) return err; } +static bool keep_in_base_btf(struct object *obj, const struct btf_type *t) +{ + const char *name; + + switch (btf_kind(t)) { + case BTF_KIND_FUNC: + name = btf__name_by_offset(obj->btf, t->name_off); + return !obj->efile.func_symbols_collected || has_func_symbol(obj, name) || + btf_id__find(&obj->funcs, name); + case BTF_KIND_FUNC_PROTO: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: + return false; + default: + /* + * Ordinary types stay in base btf even if only inline records use them. + * Should we relax this? + */ + return true; + } +} + +/* Mark which types should go to .BTF.inline */ +static int color_btf(struct object *obj, __u8 *colors) +{ + struct btf *btf = obj->btf; + const struct btf *base = btf__base_btf(btf); + __u32 start_id = base ? btf__type_cnt(base) : 1; + __u32 type_cnt = btf__type_cnt(btf), i; + __u32 *worklist; + + worklist = malloc((type_cnt - start_id ?: 1) * sizeof(*worklist)); + if (!worklist) + return -ENOMEM; + /* First, mark all types reachable from LOC BTF entries. */ + for (i = start_id; i < type_cnt; i++) { + const struct btf_type *t = btf__type_by_id(btf, i); + + if (!btf_is_locsec(t) && !btf_is_loc_proto(t) && !btf_is_loc_param(t)) + continue; + btf_mark_reachable(btf, i, BTF_COLOR_LOC, colors, worklist); + } + /* Next, traverse non-marked types converting some LOC markings to SHARED. */ + for (i = start_id; i < type_cnt; i++) { + const struct btf_type *t = btf__type_by_id(btf, i); + + if (colors[i] == BTF_COLOR_LOC && !keep_in_base_btf(obj, t)) + continue; + btf_mark_reachable(btf, i, BTF_COLOR_MAIN, colors, worklist); + } + free(worklist); + return 0; +} + +static int extract_inline_btf(struct object *obj) +{ + struct btf *btf = obj->btf, *main_btf, *inline_btf; + __u8 *colors; + int err; + + colors = calloc(btf__type_cnt(btf), sizeof(*colors)); + if (!colors) + return -ENOMEM; + err = color_btf(obj, colors); + if (err) + goto out; + err = btf_split_by_color(btf, colors, &main_btf, &inline_btf); + if (!err) { + obj->orig_btf = btf; + obj->btf = main_btf; + obj->inline_btf = inline_btf; + } +out: + free(colors); + return err; +} + /* * Sort types by name in ascending order resulting in all * anonymous types being placed before named types. @@ -1583,14 +1920,14 @@ static int finalize_btf(struct object *obj) err = btf__dedup(obj->btf, NULL); if (err) { pr_err("FAILED to dedup BTF: %s\n", strerror(errno)); - goto out_err; + return err; } if (obj->base_btf && obj->distill_base) { err = btf__distill_base(obj->btf, &base_btf, &btf); if (err) { pr_err("FAILED to distill base BTF: %s\n", strerror(errno)); - goto out_err; + return err; } btf__free(obj->base_btf); @@ -1602,18 +1939,18 @@ static int finalize_btf(struct object *obj) err = sort_btf_by_name(obj->btf); if (err) { pr_err("FAILED to sort BTF: %s\n", strerror(errno)); - goto out_err; + return err; + } + + if (obj->extract_inline) { + err = extract_inline_btf(obj); + if (err) { + pr_err("FAILED to extract inline BTF: %s\n", strerror(-err)); + return err; + } } return 0; - -out_err: - btf__free(base_btf); - btf__free(btf); - obj->base_btf = NULL; - obj->btf = NULL; - - return err; } static inline int make_out_path(char *buf, u32 buf_sz, const char *in_path, const char *suffix) @@ -1740,7 +2077,7 @@ static int patch_btfids(const char *btfids_path, const char *elf_path) static const char * const resolve_btfids_usage[] = { "resolve_btfids [<options>] <ELF object>", - "resolve_btfids --patch_btfids <.BTF_ids file> <ELF object>", + "resolve_btfids --patch_btfids <.BTF_ids file> [--btf_link <section:module:btf-file>] <ELF object>", NULL }; @@ -1758,6 +2095,7 @@ int main(int argc, const char **argv) .sets = RB_ROOT, }; const char *btfids_path = NULL; + struct btf_links btf_links = {}; bool fatal_warnings = false; bool resolve_btfids = true; char out_path[PATH_MAX]; @@ -1773,6 +2111,10 @@ int main(int argc, const char **argv) "turn warnings into errors"), OPT_BOOLEAN(0, "distill_base", &obj.distill_base, "distill --btf_base and emit .BTF.base section data"), + OPT_BOOLEAN(0, "inline", &obj.extract_inline, + "extract location BTF into a .BTF.inline file"), + OPT_CALLBACK(0, "btf_link", &btf_links, "section:module:btf-file", + "patch a BTF link with --patch_btfids", parse_btf_link), OPT_STRING(0, "patch_btfids", &btfids_path, "file", "path to .BTF_ids section data blob to patch into ELF file"), OPT_END() @@ -1783,14 +2125,34 @@ int main(int argc, const char **argv) PARSE_OPT_STOP_AT_NON_OPTION); if (argc != 1) usage_with_options(resolve_btfids_usage, btfid_options); + if (btf_links.cnt && !btfids_path) { + pr_err("--btf_link requires --patch_btfids\n"); + goto out; + } obj.path = argv[0]; - if (btfids_path) - return patch_btfids(btfids_path, obj.path); + if (btfids_path) { + err = patch_btfids(btfids_path, obj.path); + if (err || !btf_links.cnt) + goto out; + if (elf_collect(&obj)) { + err = -EINVAL; + goto out; + } + for (u32 i = 0; i < btf_links.cnt; i++) { + struct btf_link *link = &btf_links.links[i]; + err = patch_btf_link(&obj, link); + if (err) + break; + } + goto out; + } if (elf_collect(&obj)) goto out; + if (obj.extract_inline && collect_func_symbols(&obj)) + goto out; /* * We did not find .BTF_ids section or symbols section, @@ -1841,18 +2203,32 @@ int main(int argc, const char **argv) if (err) goto out; } + if (obj.inline_btf) { + err = make_out_path(out_path, sizeof(out_path), obj.path, BTF_ELF_SEC ".inline"); + err = err ?: dump_raw_btf(obj.inline_btf, out_path); + if (err) + goto out; + } - if (!(fatal_warnings && warnings)) - err = 0; + err = fatal_warnings && warnings ? -1 : 0; out: - btf__free(obj.base_btf); + free_btf_links(&btf_links); + /* + * Free BTFs in reverse dependency order. inline_btf borrows obj.btf; + * obj.btf can borrow a base owned by orig_btf or obj.base_btf, and + * orig_btf can also borrow obj.base_btf. + */ + btf__free(obj.inline_btf); btf__free(obj.btf); + btf__free(obj.orig_btf); + btf__free(obj.base_btf); btf_id__free_all(&obj.structs); btf_id__free_all(&obj.unions); btf_id__free_all(&obj.typedefs); btf_id__free_all(&obj.funcs); btf_id__free_all(&obj.sets); free(obj.addr_syms); + free(obj.efile.func_symbols); if (obj.efile.elf) { elf_end(obj.efile.elf); close(obj.efile.fd);
diff --git a/tools/dma/Makefile b/tools/dma/Makefile index e4abf37..20ae7f0 100644 --- a/tools/dma/Makefile +++ b/tools/dma/Makefile
@@ -7,7 +7,13 @@ # isn't set and when invoked from selftests build, where srctree # is set to ".". building_out_of_srctree is undefined for in srctree # builds +ifeq ($(srctree),) +update_srctree := 1 +endif ifndef building_out_of_srctree +update_srctree := 1 +endif +ifeq ($(update_srctree),1) srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif @@ -38,8 +44,8 @@ FORCE: DMA_MAP_BENCHMARK = dma_map_benchmark -$(DMA_MAP_BENCHMARK): prepare FORCE - $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $(DMA_MAP_BENCHMARK) +$(OUTPUT)$(DMA_MAP_BENCHMARK): prepare FORCE + $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $@ clean: rm -f $(ALL_PROGRAMS)
diff --git a/tools/include/asm-generic/io.h b/tools/include/asm-generic/io.h index e5a0b07..0d89dec 100644 --- a/tools/include/asm-generic/io.h +++ b/tools/include/asm-generic/io.h
@@ -479,4 +479,32 @@ static inline void writesq(volatile void __iomem *addr, const void *buffer, } #endif +/* + * ioread/iowrite for big-endian MMIO registers. + */ + +#ifndef ioread32be +#define ioread32be ioread32be +static inline u32 ioread32be(const volatile void __iomem *addr) +{ + return bswap_32(readl(addr)); +} +#endif + +#ifndef iowrite32be +#define iowrite32be iowrite32be +static inline void iowrite32be(u32 value, volatile void __iomem *addr) +{ + writel(bswap_32(value), addr); +} +#endif + +#ifndef iowrite64be +#define iowrite64be iowrite64be +static inline void iowrite64be(u64 value, volatile void __iomem *addr) +{ + writeq(bswap_64(value), addr); +} +#endif + #endif /* _TOOLS_ASM_GENERIC_IO_H */
diff --git a/tools/include/asm/barrier.h b/tools/include/asm/barrier.h index 0c21678..e7e0c7d 100644 --- a/tools/include/asm/barrier.h +++ b/tools/include/asm/barrier.h
@@ -47,6 +47,14 @@ # define smp_mb() mb() #endif +#ifndef dma_rmb +# define dma_rmb() rmb() +#endif + +#ifndef dma_wmb +# define dma_wmb() wmb() +#endif + #ifndef smp_store_release # define smp_store_release(p, v) \ do { \
diff --git a/tools/include/linux/stddef.h b/tools/include/linux/stddef.h new file mode 100644 index 0000000..99182ea --- /dev/null +++ b/tools/include/linux/stddef.h
@@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TOOLS_LINUX_STDDEF_H +#define _TOOLS_LINUX_STDDEF_H + +#include_next <linux/stddef.h> + +#define DECLARE_FLEX_ARRAY(TYPE, NAME) \ + __DECLARE_FLEX_ARRAY(TYPE, NAME) + +#endif /* _TOOLS_LINUX_STDDEF_H */
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 732b35c..f29b0aa 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h
@@ -56,6 +56,19 @@ enum bpf_cond_pseudo_jmp { BPF_MAY_GOTO = 0, + /* + * Follows a call of a bpf subprog. Taken when the subprog left with + * BPF_EXIT that has src_reg == BPF_UNWIND_EXIT. + */ + BPF_UNWIND_GOTO = 1, +}; + +enum bpf_pseudo_exit { + /* + * When bpf_exit->src_reg == BPF_UNWIND_EXIT, it's the same as bpf_exit, + * except that BPF_UNWIND_GOTO after the call is taken. + */ + BPF_UNWIND_EXIT = 1, }; /* Register numbers */ @@ -936,6 +949,33 @@ union bpf_iter_link_info { * 0 on success or -1 if an error occurred (in which case, * *errno* is set appropriately). * + * BPF_PROG_STREAM_OPEN + * Description + * Open a file descriptor for one of the BPF streams associated + * with the program identified by *prog_fd*. The stream is selected + * by *stream_id*. + * + * The returned file descriptor supports **read**\ (2) and + * **poll**\ (2). Reads block while the stream is empty unless + * **BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking + * read of an empty stream fails with **EAGAIN**. + * + * **poll**\ (2) reports **POLLIN** when data is available and + * **POLLHUP** once the program has been freed, that is, after every + * reference to it, including links and other file descriptors, has + * been dropped. Hangup may lag the final release because program + * teardown is deferred. Buffered data remains readable after + * **POLLHUP** and a read returns zero after all such data has been + * consumed. + * + * The file descriptor is read-only and has the close-on-exec flag + * set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**. + * *flags* may only contain **BPF_F_STREAM_NONBLOCK**. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -993,6 +1033,7 @@ enum bpf_cmd { BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_STREAM_OPEN, __MAX_BPF_CMD, BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */ }; @@ -1316,6 +1357,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ @@ -1524,6 +1571,11 @@ enum { BPF_STREAM_STDERR = 2, }; +/* flags for BPF_PROG_STREAM_OPEN command */ +enum { + BPF_F_STREAM_NONBLOCK = (1U << 0), +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1756,7 +1808,7 @@ union bpf_attr { __u32 prog_cnt; __u32 count; }; - __u32 :32; + __u32 type_id; /* output: per-program attach_flags. * not allowed to be set during effective query. */ @@ -1950,6 +2002,12 @@ union bpf_attr { __u32 flags; } prog_assoc_struct_ops; + struct { + __u32 prog_fd; + __u32 stream_id; + __u32 flags; + } prog_stream_open; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF @@ -4867,15 +4925,18 @@ union bpf_attr { * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) + * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags) * Description * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). + * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). For the + * **bpf_tcp_ops** struct_ops, this helper can be called from the + * **parse_hdr**\ () and **write_hdr_opt**\ () operators. * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. + * If *flags* is 0, it will search the option from the packet + * associated with the current operation. For + * **BPF_PROG_TYPE_SOCK_OPS**, the comment in + * **struct bpf_sock_ops** has details on what skb_data + * contains under different *op*. * * The first byte of the *searchby_res* specifies the * kind that it wants to search. @@ -4908,6 +4969,8 @@ union bpf_attr { * * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the * saved_syn packet or the just-received syn packet. + * Not supported by the **bpf_tcp_ops** struct_ops, which + * rejects all flags. * * Return * > 0 when found, the header option is copied to *searchby_res*. @@ -4928,9 +4991,9 @@ union bpf_attr { * packet. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) + * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags) * Description * Store header option. The data will be copied * from buffer *from* with length *len* to the TCP header. @@ -4946,7 +5009,9 @@ union bpf_attr { * by searching the same option in the outgoing skb. * * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. + * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the + * **write_hdr_opt**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4961,9 +5026,9 @@ union bpf_attr { * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) + * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags) * Description * Reserve *len* bytes for the bpf header option. The * space will be used by **bpf_store_hdr_opt**\ () later in @@ -4973,7 +5038,9 @@ union bpf_attr { * the total number of bytes will be reserved. * * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. + * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the + * **hdr_opt_len**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4983,7 +5050,7 @@ union bpf_attr { * **-ENOSPC** if there is not enough space in the header. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) * Description @@ -6890,6 +6957,8 @@ struct bpf_link_info { } xdp; struct { __u32 map_id; + __u32 :32; + __u64 cgroup_id; } struct_ops; struct { __u32 pf; @@ -7284,6 +7353,31 @@ enum { */ }; +/* + * Most callbacks of struct bpf_tcp_ops are placed in the slow + * path (e.g., one-shot connection setup or unlikely events like + * timers) and are invoked simply by defining non-NULL callbacks. + * + * Callbacks in the fast path, however, would incur noticeable + * overhead even when set to NULL, so they are disabled by default + * and must be explicitly enabled per socket via bpf_tcp_ops_set_flags(). + * + * The flags are per-socket and shared by all effective bpf_tcp_ops + * programs. + */ +enum { + /* .rtt() */ + BPF_TCP_OPS_FLAG_RTT = (1 << 0), + /* .parse_hdr() */ + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL = (1 << 1), + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), + /* .hdr_opt_len() and .write_hdr_opt() */ + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), + /* .enqueue_rcvq() and .dequeue_rcvq() */ + BPF_TCP_OPS_FLAG_RCVQ = (1 << 4), + BPF_TCP_OPS_FLAG_ALL = (1 << 5) - 1, +}; + /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect * changes between the TCP and BPF versions. Ideally this should never happen. * If it does, we need to add code to convert them before calling @@ -7600,6 +7694,10 @@ struct bpf_task_work { __u64 __opaque; } __attribute__((aligned(8))); +struct bpf_rcu_head { + __u64 __opaque[6]; +} __attribute__((aligned(8))); + struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8)));
diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h index 618167c..11d6587 100644 --- a/tools/include/uapi/linux/btf.h +++ b/tools/include/uapi/linux/btf.h
@@ -54,8 +54,8 @@ struct btf_type { * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64 + * and LOC_PARAM. "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. @@ -92,7 +92,9 @@ enum { BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - + BTF_KIND_LOC_PARAM = 20, /* Location parameter information */ + BTF_KIND_LOC_PROTO = 21, /* Location prototype for site */ + BTF_KIND_LOCSEC = 22, /* Location section */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, }; @@ -212,4 +214,79 @@ struct btf_enum64 { __u32 val_hi32; }; +/* + * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param" + * that contains flags specifying the contents of the vlen-specified + * number of 4-byte values that follow. + */ +struct btf_loc_param { + __u32 flags; + __u32 values[]; +}; + +/* + * The combination of size, vlen and flags gives us the means to interpret + * the following vlen-specified set of 4-byte values: + * + * - a BTF_LOC_PARAM_CONST is a constant value; combination + * of size, vlen and _SIGNED flag determines it. If the value requires + * 64 bits it is stored in {lo,hi} order. + * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be + * normalized with respect to kernel/module base address. + * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number; + * with vlen 2 it is a multi-register parameter. Register numbers are + * numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since + * DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer + * to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for + * the frame base register to avoid collisions. + * - a _REG | OFFSET describes an address without dereferencing it. + * - a _REG | DEREF with vlen 1 dereferences the value in the register + * number specified. + * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number + * in the first 4-byte value and the offset in the remainder. + * In the case of REG and OFFSET combinations, the OFFSET has the width + * of the value words while the type size describes the represented parameter, + * so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be + * an 8-byte register dereference with signed 4-byte offset, since the vlen 2 + * values consist of a register value and the signed value. + */ +enum btf_loc_param_flags { + BTF_LOC_PARAM_SIGNED = 0x1, + BTF_LOC_PARAM_CONST = 0x2, + BTF_LOC_PARAM_ADDR = 0x4, + BTF_LOC_PARAM_REG = 0x8, + BTF_LOC_PARAM_DEREF = 0x10, + BTF_LOC_PARAM_OFFSET = 0x20, +}; + +enum { + BTF_LOC_PARAM_FBREG = 0xffffffff +}; + +/* + * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate + * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a + * vlen-specified number of __u32 BTF type ids which specify the associated + * BTF_KIND_LOC_PARAM for each function parameter associated with the + * location. The type should either be 0 (no location info) or point at + * a BTF_KIND_LOC_PARAM. + */ + +/* + * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc" + * containing location site-specific information for a specific ELF section; + * for example locations in ".text" are in a LOCSEC named "inline.text". + * + * - function (func) + * - location prototype type id (loc_proto) + * - address offset (offset) relative to the runtime base address of the + * ELF section associated with the LOCSEC + */ + +struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; +}; + #endif /* _UAPI__LINUX_BTF_H__ */
diff --git a/tools/include/uapi/linux/types.h b/tools/include/uapi/linux/types.h index 85aa327..6acf205 100644 --- a/tools/include/uapi/linux/types.h +++ b/tools/include/uapi/linux/types.h
@@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_LINUX_TYPES_H #define _UAPI_LINUX_TYPES_H
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 96819c0..b49822d 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c
@@ -934,6 +934,7 @@ int bpf_link_create(int prog_fd, int target_fd, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: + case BPF_STRUCT_OPS: relative_fd = OPTS_GET(opts, cgroup.relative_fd, 0); relative_id = OPTS_GET(opts, cgroup.relative_id, 0); if (relative_fd && relative_id) @@ -1056,6 +1057,7 @@ int bpf_prog_query_opts(int target, enum bpf_attach_type type, attr.query.attach_type = type; attr.query.query_flags = OPTS_GET(opts, query_flags, 0); attr.query.count = OPTS_GET(opts, count, 0); + attr.query.type_id = OPTS_GET(opts, type_id, 0); attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); attr.query.link_ids = ptr_to_u64(OPTS_GET(opts, link_ids, NULL)); attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); @@ -1464,6 +1466,25 @@ int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len, return libbpf_err_errno(err); } +int bpf_prog_stream_open(int prog_fd, __u32 stream_id, + const struct bpf_prog_stream_open_opts *opts) +{ + const size_t attr_sz = offsetofend(union bpf_attr, prog_stream_open); + union bpf_attr attr; + int fd; + + if (!OPTS_VALID(opts, bpf_prog_stream_open_opts)) + return libbpf_err(-EINVAL); + + memset(&attr, 0, attr_sz); + attr.prog_stream_open.prog_fd = prog_fd; + attr.prog_stream_open.stream_id = stream_id; + attr.prog_stream_open.flags = OPTS_GET(opts, flags, 0); + + fd = sys_bpf_fd(BPF_PROG_STREAM_OPEN, &attr, attr_sz); + return libbpf_err_errno(fd); +} + int bpf_prog_assoc_struct_ops(int prog_fd, int map_fd, struct bpf_prog_assoc_struct_ops_opts *opts) {
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 7534a59..826d9cc 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h
@@ -637,9 +637,10 @@ struct bpf_prog_query_opts { __u32 *link_ids; __u32 *link_attach_flags; __u64 revision; + __u32 type_id; size_t :0; }; -#define bpf_prog_query_opts__last_field revision +#define bpf_prog_query_opts__last_field type_id /** * @brief **bpf_prog_query_opts()** queries the BPF programs and BPF links @@ -758,10 +759,34 @@ struct bpf_prog_stream_read_opts { * * @return The number of bytes read, on success; negative error code, otherwise * (errno is also set to the error code) + * + * For blocking reads and polling, prefer **bpf_prog_stream_open**. */ LIBBPF_API int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len, struct bpf_prog_stream_read_opts *opts); +struct bpf_prog_stream_open_opts { + size_t sz; + __u32 flags; + size_t :0; +}; +#define bpf_prog_stream_open_opts__last_field flags + +/** + * @brief **bpf_prog_stream_open** opens a file descriptor for a BPF stream of + * a given BPF program. + * + * @param prog_fd FD for the BPF program whose BPF stream is to be opened. + * @param stream_id ID of the BPF stream to be opened. + * @param opts optional options, can be NULL. BPF_F_STREAM_NONBLOCK requests a + * non-blocking descriptor. + * + * @return A new stream FD, on success; negative error code, otherwise (errno + * is also set to the error code) + */ +LIBBPF_API int bpf_prog_stream_open(int prog_fd, __u32 stream_id, + const struct bpf_prog_stream_open_opts *opts); + struct bpf_prog_assoc_struct_ops_opts { size_t sz; __u32 flags;
diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 6c5ad6c..206adf2 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h
@@ -51,9 +51,17 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; + /* + * Maps with pointers to functions, that programs get their own copies + * of, take slots in fd_array after nr_obj_maps maps of the object. + */ + __u32 nr_obj_maps; + __u32 nr_func_ptr_maps; + __u32 max_func_ptr_maps; }; -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps); int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps); void bpf_gen__free(struct bpf_gen *gen); void bpf_gen__load_btf(struct bpf_gen *gen, const void *raw_data, __u32 raw_size); @@ -68,6 +76,9 @@ void bpf_gen__prog_load(struct bpf_gen *gen, void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *value, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, bool is_typeless, bool is_ld64, int kind, int insn_idx);
diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 9d160b5..b37b727 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h
@@ -249,12 +249,20 @@ enum libbpf_tristate { ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) /* + * Format strings are in a section of their own. When global data of the object + * is in arena (the object has .arena.data section) libbpf keeps .rodata.str* + * sections in read-only maps, which is where helpers and kfuncs take strings + * from. + */ +#define ___bpf_fmt_sec __attribute__((section(".rodata.str"))) + +/* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ #define BPF_SEQ_PRINTF(seq, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -272,7 +280,7 @@ enum libbpf_tristate { */ #define BPF_SNPRINTF(out, out_size, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -287,7 +295,7 @@ enum libbpf_tristate { #ifdef BPF_NO_GLOBAL_DATA #define BPF_PRINTK_FMT_MOD #else -#define BPF_PRINTK_FMT_MOD static const +#define BPF_PRINTK_FMT_MOD static const ___bpf_fmt_sec #endif #define __bpf_printk(fmt, ...) \ @@ -303,7 +311,7 @@ enum libbpf_tristate { */ #define __bpf_vprintk(fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -317,7 +325,7 @@ enum libbpf_tristate { #define bpf_stream_printk(stream_id, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 8417de9..d2ff6f5 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c
@@ -57,6 +57,9 @@ static struct btf_layout layouts[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = { sizeof(struct btf_decl_tag), 0, 0 }, [BTF_KIND_TYPE_TAG] = { 0, 0, 0 }, [BTF_KIND_ENUM64] = { 0, sizeof(struct btf_enum64), 0 }, +[BTF_KIND_LOC_PARAM] = { sizeof(struct btf_loc_param), sizeof(__u32), 0 }, +[BTF_KIND_LOC_PROTO] = { 0, sizeof(__u32), 0 }, +[BTF_KIND_LOCSEC] = { 0, sizeof(struct btf_loc), 0 }, }; struct btf { @@ -486,6 +489,12 @@ static int btf_type_size(const struct btf *btf, const struct btf_type *t) return base_size + vlen * sizeof(struct btf_var_secinfo); case BTF_KIND_DECL_TAG: return base_size + sizeof(struct btf_decl_tag); + case BTF_KIND_LOC_PARAM: + return base_size + sizeof(struct btf_loc_param) + vlen * sizeof(__u32); + case BTF_KIND_LOC_PROTO: + return base_size + vlen * sizeof(__u32); + case BTF_KIND_LOCSEC: + return base_size + vlen * sizeof(struct btf_loc); default: return btf_type_size_unknown(btf, t); } @@ -501,12 +510,15 @@ static void btf_bswap_type_base(struct btf_type *t) static int btf_bswap_type_rest(struct btf_type *t) { struct btf_var_secinfo *v; + struct btf_loc_param *lp; struct btf_enum64 *e64; struct btf_member *m; struct btf_array *a; struct btf_param *p; struct btf_enum *e; + struct btf_loc *l; __u32 vlen = btf_vlen(t); + __u32 *d; int i; switch (btf_kind(t)) { @@ -569,6 +581,23 @@ static int btf_bswap_type_rest(struct btf_type *t) case BTF_KIND_DECL_TAG: btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); return 0; + case BTF_KIND_LOC_PARAM: + lp = btf_loc_param(t); + lp->flags = bswap_32(lp->flags); + for (i = 0, d = (__u32 *)(lp + 1); i < vlen; i++, d++) + *d = bswap_32(*d); + return 0; + case BTF_KIND_LOC_PROTO: + for (i = 0, d = btf_loc_proto_params(t); i < vlen; i++, d++) + *d = bswap_32(*d); + return 0; + case BTF_KIND_LOCSEC: + for (i = 0, l = btf_locsec_locs(t); i < vlen; i++, l++) { + l->func = bswap_32(l->func); + l->loc_proto = bswap_32(l->loc_proto); + l->offset = bswap_32(l->offset); + } + return 0; default: pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); return -EINVAL; @@ -745,6 +774,33 @@ static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __ } break; } + case BTF_KIND_LOC_PARAM: + break; + case BTF_KIND_LOC_PROTO: { + __u32 *p = btf_loc_proto_params(t); + + n = btf_vlen(t); + for (i = 0; i < n; i++, p++) { + err = btf_validate_id(btf, *p, id); + if (err) + return err; + } + break; + } + case BTF_KIND_LOCSEC: { + const struct btf_loc *l = btf_locsec_locs(t); + + n = btf_vlen(t); + for (i = 0; i < n; i++, l++) { + if (!err) + err = btf_validate_id(btf, l->func, id); + if (!err) + err = btf_validate_id(btf, l->loc_proto, id); + if (err) + return err; + } + break; + } default: /* Kind may be represented in kind layout information. */ if (btf_type_size_unknown(btf, t) < 0) { @@ -3344,6 +3400,243 @@ int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1); } +/* + * Append new BTF_KIND_LOC_PARAM with specified size and flags. Values are + * added via btf__add_loc_param_value(). + * + * - *byte_sz* - size of the type, in bytes (1 through 16); + * - *flags* - combination of enum btf_loc_param_flags values + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_loc_param(struct btf *btf, __u32 byte_sz, __u32 flags) +{ + struct btf_loc_param *p; + struct btf_type *t; + int sz, err; + + if (!byte_sz || byte_sz > 16) + return libbpf_err(-EINVAL); + + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(struct btf_type) + sizeof(*p); + t = btf_add_type_mem(btf, sz); + if (!t) + return libbpf_err(-ENOMEM); + + t->name_off = 0; + t->info = btf_type_info(BTF_KIND_LOC_PARAM, 0, 0); + t->size = byte_sz; + + p = btf_loc_param(t); + p->flags = flags; + + return btf_commit_type(btf, sz); +} + +/* + * Append *value* to existing BTF_KIND_LOC_PARAM. + * + * Returns: + * - 0 on success + * - <0, on error. + */ +int btf__add_loc_param_value(struct btf *btf, __u32 value) +{ + struct btf_type *t; + int sz, err; + __u32 *v; + + /* last type should be BTF_KIND_LOC_PARAM */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_loc_param(t)) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(value); + v = btf_add_type_mem(btf, sz); + if (!v) + return libbpf_err(-ENOMEM); + *v = value; + + /* update parent type's vlen */ + t = btf_last_type(btf); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); + + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); + return 0; +} + +/* + * Append new BTF_KIND_LOC_PROTO + * + * The prototype is then populated with 0 or more BTF_KIND_LOC_PARAMs via + * btf__add_loc_proto_param(); similar to how btf__add_func_param() adds + * parameters to a FUNC_PROTO. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_loc_proto(struct btf *btf) +{ + struct btf_type *t; + int err; + + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + t = btf_add_type_mem(btf, sizeof(struct btf_type)); + if (!t) + return libbpf_err(-ENOMEM); + + t->name_off = 0; + t->info = btf_type_info(BTF_KIND_LOC_PROTO, 0, 0); + t->size = 0; + + return btf_commit_type(btf, sizeof(struct btf_type)); +} + +/* + * Append a BTF_KIND_LOC_PARAM id/0 to BTF_KIND_LOC_PROTO. + * + * Returns: + * - 0 on success; + * - <0, on error. + + */ +int btf__add_loc_proto_param(struct btf *btf, __u32 id) +{ + struct btf_type *t; + int sz, err; + __u32 *p; + + if (validate_type_id(id)) + return libbpf_err(-EINVAL); + + /* last type should be BTF_KIND_LOC_PROTO */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_loc_proto(t)) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(__u32); + p = btf_add_type_mem(btf, sz); + if (!p) + return libbpf_err(-ENOMEM); + *p = id; + + /* update parent type's vlen */ + t = btf_last_type(btf); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); + + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); + return 0; +} + +/* + * Append new BTF_KIND_LOCSEC type with: + * - *name* - non-empty/non-NULL name; + * + * Location section is initially empty. Location info can be added with + * btf__add_locsec_loc() calls, after btf__add_locsec() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ + +int btf__add_locsec(struct btf *btf, const char *name) +{ + struct btf_type *t; + int name_off = 0; + int err; + + /* non-empty name */ + if (str_is_empty(name)) + return libbpf_err(-EINVAL); + + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + t = btf_add_type_mem(btf, sizeof(struct btf_type)); + if (!t) + return libbpf_err(-ENOMEM); + + name_off = btf__add_str(btf, name); + if (name_off < 0) + return libbpf_err(name_off); + t->name_off = name_off; + t->info = btf_type_info(BTF_KIND_LOCSEC, 0, 0); + t->size = 0; + + return btf_commit_type(btf, sizeof(struct btf_type)); +} + +int btf__add_locsec_loc(struct btf *btf, __u32 func, __u32 loc_proto, + __u32 offset) +{ + struct btf_type *t; + struct btf_loc *l; + int sz, err; + + if (validate_type_id(func) || validate_type_id(loc_proto)) + return libbpf_err(-EINVAL); + + /* last type should be BTF_KIND_LOCSEC */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_locsec(t)) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(*l); + l = btf_add_type_mem(btf, sz); + if (!l) + return libbpf_err(-ENOMEM); + + l->func = func; + l->loc_proto = loc_proto; + l->offset = offset; + + /* update parent type's vlen */ + t = btf_last_type(btf); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); + + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); + return 0; +} + struct btf_ext_sec_info_param { __u32 off; __u32 len; @@ -3364,7 +3657,7 @@ static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, { const struct btf_ext_info_sec *sinfo; struct btf_ext_info *ext_info; - __u32 info_left, record_size; + __u32 data_left, info_left, record_size; size_t sec_cnt = 0; void *info; @@ -3377,16 +3670,20 @@ static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, return -EINVAL; } - /* The start of the info sec (including the __u32 record_size). */ - info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; - info_left = ext_sec->len; + if (btf_ext->hdr->hdr_len > btf_ext->data_size) + return -EINVAL; - if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { + data_left = btf_ext->data_size - btf_ext->hdr->hdr_len; + if (ext_sec->off > data_left || ext_sec->len > data_left - ext_sec->off) { pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", ext_sec->desc, ext_sec->off, ext_sec->len); return -EINVAL; } + /* The start of the info sec (including the __u32 record_size). */ + info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; + info_left = ext_sec->len; + /* At least a record size */ if (info_left < sizeof(__u32)) { pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); @@ -4110,8 +4407,8 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_o for (i = 1; i < type_cnt; i++) { struct btf_type *t = btf_type_by_id(d->btf, i); - /* VAR and DATASEC are never deduped and are self-canonical */ - if (btf_is_var(t) || btf_is_datasec(t)) + /* VAR, DATASEC and LOCSEC are never deduped and are self-canonical */ + if (btf_is_var(t) || btf_is_datasec(t) || btf_is_locsec(t)) d->map[i] = i; else d->map[i] = BTF_UNPROCESSED_ID; @@ -4391,6 +4688,47 @@ static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) btf_is_any_enum(t1) && btf_is_any_enum(t2); } +static long btf_hash_loc_param(struct btf_type *t) +{ + struct btf_loc_param *p = btf_loc_param(t); + long h = btf_hash_common(t); + int i, vlen = btf_vlen(t); + + h = hash_combine(h, p->flags); + + for (i = 0; i < vlen; i++) + h = hash_combine(h, p->values[i]); + return h; +} + +static long btf_hash_loc_proto(struct btf_type *t) +{ + __u32 *p = btf_loc_proto_params(t); + long h = btf_hash_common(t); + int i, vlen = btf_vlen(t); + + for (i = 0; i < vlen; i++, p++) + h = hash_combine(h, *p); + return h; +} + +static bool btf_equal_loc_param(struct btf_type *t1, struct btf_type *t2) +{ + struct btf_loc_param *p1 = btf_loc_param(t1); + struct btf_loc_param *p2 = btf_loc_param(t2); + int i, vlen = btf_vlen(t1); + + if (!btf_equal_common(t1, t2)) + return false; + if (p1->flags != p2->flags) + return false; + for (i = 0; i < vlen; i++) { + if (p1->values[i] != p2->values[i]) + return false; + } + return true; +} + /* * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, * as referenced type IDs equivalence is established separately during type @@ -4585,7 +4923,8 @@ static int btf_dedup_prep(struct btf_dedup *d) switch (btf_kind(t)) { case BTF_KIND_VAR: case BTF_KIND_DATASEC: - /* VAR and DATASEC are never hash/deduplicated */ + case BTF_KIND_LOCSEC: + /* VAR DATASEC and LOCSEC are never hash/deduplicated */ continue; case BTF_KIND_CONST: case BTF_KIND_VOLATILE: @@ -4618,6 +4957,12 @@ static int btf_dedup_prep(struct btf_dedup *d) case BTF_KIND_FUNC_PROTO: h = btf_hash_fnproto(t); break; + case BTF_KIND_LOC_PARAM: + h = btf_hash_loc_param(t); + break; + case BTF_KIND_LOC_PROTO: + h = btf_hash_loc_proto(t); + break; default: pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); return -EINVAL; @@ -4660,6 +5005,8 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_DATASEC: case BTF_KIND_DECL_TAG: case BTF_KIND_TYPE_TAG: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: return 0; case BTF_KIND_INT: @@ -4709,6 +5056,18 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) } break; + case BTF_KIND_LOC_PARAM: + h = btf_hash_loc_param(t); + for_each_dedup_cand(d, hash_entry, h) { + cand_id = hash_entry->value; + cand = btf_type_by_id(d->btf, cand_id); + if (btf_equal_loc_param(t, cand)) { + new_id = cand_id; + break; + } + } + break; + default: return -EINVAL; } @@ -4827,7 +5186,7 @@ static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2, return false; a1 = btf_array(t1); - a2 = btf_array(t1); + a2 = btf_array(t2); if (a1->index_type != a2->index_type && !btf_dedup_identical_types(d, a1->index_type, a2->index_type, depth - 1)) @@ -5141,6 +5500,13 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, return 1; } + case BTF_KIND_LOC_PARAM: + return btf_equal_loc_param(cand_type, canon_type); + + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: + return 0; + default: return -EINVAL; } @@ -5485,6 +5851,37 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) break; } + case BTF_KIND_LOC_PROTO: { + __u32 *p1, *p2; + __u32 i, vlen; + + p1 = btf_loc_proto_params(t); + vlen = btf_vlen(t); + + for (i = 0; i < vlen; i++, p1++) { + ref_type_id = btf_dedup_ref_type(d, *p1); + if (ref_type_id < 0) + return ref_type_id; + *p1 = ref_type_id; + } + + h = btf_hash_loc_proto(t); + for_each_dedup_cand(d, hash_entry, h) { + cand_id = hash_entry->value; + cand = btf_type_by_id(d->btf, cand_id); + if (!btf_equal_common(t, cand)) + continue; + vlen = btf_vlen(cand); + p1 = btf_loc_proto_params(t); + p2 = btf_loc_proto_params(cand); + if (memcmp(p1, p2, vlen * sizeof(__u32)) == 0) { + new_id = cand_id; + break; + } + } + break; + } + default: return -EINVAL; } @@ -5966,8 +6363,11 @@ static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_VOLATILE: + case BTF_KIND_FUNC: case BTF_KIND_FUNC_PROTO: case BTF_KIND_TYPE_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: dist->id_map[*id] = *id; break; default: @@ -5993,7 +6393,7 @@ static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) static int btf_add_distilled_types(struct btf_distill *dist) { - bool adding_to_base = dist->pipe.dst->start_id == 1; + bool adding_to_base = dist->pipe.dst->base_btf == NULL; int id = btf__type_cnt(dist->pipe.dst); struct btf_type *t; int i, err = 0; @@ -6061,8 +6461,12 @@ static int btf_add_distilled_types(struct btf_distill *dist) case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_VOLATILE: + case BTF_KIND_FUNC: case BTF_KIND_FUNC_PROTO: case BTF_KIND_TYPE_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: /* All other types are added to split BTF. */ if (adding_to_base) continue; @@ -6236,7 +6640,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) int btf__relocate(struct btf *btf, const struct btf *base_btf) { - int err = btf_relocate(btf, base_btf, NULL); + int err = btf_relocate(btf, base_btf, NULL, NULL); if (!err) btf->owns_base = false;
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 587172c..57f1263 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h
@@ -274,6 +274,20 @@ LIBBPF_API int btf__add_decl_tag(struct btf *btf, const char *value, int ref_typ LIBBPF_API int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, int component_idx); +/* location construction APIs */ +LIBBPF_API int btf__add_loc_param(struct btf *btf, __u32 size, __u32 flags); + +LIBBPF_API int btf__add_loc_param_value(struct btf *btf, __u32 value); + +LIBBPF_API int btf__add_loc_proto(struct btf *btf); + +LIBBPF_API int btf__add_loc_proto_param(struct btf *btf, __u32 id); + +LIBBPF_API int btf__add_locsec(struct btf *btf, const char *name); + +LIBBPF_API int btf__add_locsec_loc(struct btf *btf, __u32 func, __u32 loc_proto, + __u32 offset); + struct btf_dedup_opts { size_t sz; /* optional .BTF.ext info to dedup along the main BTF info */ @@ -431,6 +445,12 @@ btf_dump__dump_type_data(struct btf_dump *d, __u32 id, #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ #define BTF_KIND_ENUM64 19 /* Enum for up-to 64bit values */ +#define BTF_KIND_LOC_PARAM 20 /* Parameter at location */ +#define BTF_KIND_LOC_PROTO 21 /* Parameter set at location */ +#define BTF_KIND_LOCSEC 22 /* Section containing location info */ + +struct btf_loc_param; +struct btf_loc; static inline __u16 btf_kind(const struct btf_type *t) { @@ -569,6 +589,21 @@ static inline bool btf_is_any_enum(const struct btf_type *t) return btf_is_enum(t) || btf_is_enum64(t); } +static inline bool btf_is_loc_param(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_LOC_PARAM; +} + +static inline bool btf_is_loc_proto(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_LOC_PROTO; +} + +static inline bool btf_is_locsec(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_LOCSEC; +} + static inline bool btf_kind_core_compat(const struct btf_type *t1, const struct btf_type *t2) { @@ -683,6 +718,21 @@ static inline struct btf_decl_tag *btf_decl_tag(const struct btf_type *t) return (struct btf_decl_tag *)(t + 1); } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_locsec_locs(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + #ifdef __cplusplus } /* extern "C" */ #endif
diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 123c448..7dedd39 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c
@@ -77,6 +77,11 @@ struct btf_dump_data { bool is_array_char; }; +struct decl_tag_desc { + __u32 target_id; + __u32 tag_id; +}; + struct btf_dump { const struct btf *btf; btf_dump_printf_fn_t printf_fn; @@ -93,6 +98,11 @@ struct btf_dump { const char **cached_names; size_t cached_names_cap; + /* decl tags, sorted by (target ID, tag ID) */ + struct decl_tag_desc *decl_tags; + size_t decl_tag_cnt; + size_t decl_tag_cap; + /* topo-sorted list of dependent type definitions */ __u32 *emit_queue; int emit_queue_cap; @@ -143,7 +153,7 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) va_end(args); } -static int btf_dump_mark_referenced(struct btf_dump *d); +static int btf_dump_mark_referenced(struct btf_dump *d, const struct btf_type *t); static int btf_dump_resize(struct btf_dump *d); struct btf_dump *btf_dump__new(const struct btf *btf, @@ -192,9 +202,36 @@ struct btf_dump *btf_dump__new(const struct btf *btf, return libbpf_err_ptr(err); } +static int btf_dump_cmp_decl_tags(const void *a, const void *b) +{ + const struct decl_tag_desc *x = a, *y = b; + + if (x->target_id != y->target_id) + return x->target_id < y->target_id ? -1 : 1; + if (x->tag_id != y->tag_id) + return x->tag_id < y->tag_id ? -1 : 1; + return 0; +} + +static int btf_dump_push_decl_tag(struct btf_dump *d, __u32 id, const struct btf_type *t) +{ + if (libbpf_ensure_mem((void **)&d->decl_tags, &d->decl_tag_cap, + sizeof(*d->decl_tags), d->decl_tag_cnt + 1)) + return -ENOMEM; + + d->decl_tags[d->decl_tag_cnt++] = (struct decl_tag_desc) { + .target_id = t->type, + .tag_id = id, + }; + return 0; +} + static int btf_dump_resize(struct btf_dump *d) { int err, last_id = btf__type_cnt(d->btf) - 1; + size_t cnt = d->decl_tag_cnt; + const struct btf_type *t; + __u32 i; if (last_id <= d->last_id) return 0; @@ -212,13 +249,30 @@ static int btf_dump_resize(struct btf_dump *d) d->type_states[0].emit_state = EMITTED; } - /* eagerly determine referenced types for anon enums */ - err = btf_dump_mark_referenced(d); - if (err) - return err; + for (i = d->last_id + 1; i <= last_id; i++) { + t = btf__type_by_id(d->btf, i); + + err = btf_dump_mark_referenced(d, t); + if (err) + goto out_err; + + if (btf_is_decl_tag(t)) { + err = btf_dump_push_decl_tag(d, i, t); + if (err) + goto out_err; + } + } + + if (d->decl_tag_cnt != cnt) + qsort(d->decl_tags, d->decl_tag_cnt, sizeof(*d->decl_tags), + btf_dump_cmp_decl_tags); d->last_id = last_id; return 0; + +out_err: + d->decl_tag_cnt = cnt; + return err; } static void btf_dump_free_names(struct hashmap *map) @@ -251,6 +305,7 @@ void btf_dump__free(struct btf_dump *d) } } free(d->cached_names); + free(d->decl_tags); free(d->emit_queue); free(d->decl_stack); btf_dump_free_names(d->type_names); @@ -312,68 +367,64 @@ int btf_dump__dump_type(struct btf_dump *d, __u32 id) * top-level anonymous enum won't be referenced by anything, while embedded * one will. */ -static int btf_dump_mark_referenced(struct btf_dump *d) +static int btf_dump_mark_referenced(struct btf_dump *d, const struct btf_type *t) { - int i, j, n = btf__type_cnt(d->btf); - const struct btf_type *t; - __u32 vlen; + __u32 j, vlen = btf_vlen(t); - for (i = d->last_id + 1; i < n; i++) { - t = btf__type_by_id(d->btf, i); - vlen = btf_vlen(t); + switch (btf_kind(t)) { + case BTF_KIND_INT: + case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: + case BTF_KIND_FWD: + case BTF_KIND_FLOAT: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: + break; - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FWD: - case BTF_KIND_FLOAT: - break; + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + case BTF_KIND_PTR: + case BTF_KIND_TYPEDEF: + case BTF_KIND_FUNC: + case BTF_KIND_VAR: + case BTF_KIND_DECL_TAG: + case BTF_KIND_TYPE_TAG: + d->type_states[t->type].referenced = 1; + break; - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_VAR: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - d->type_states[t->type].referenced = 1; - break; + case BTF_KIND_ARRAY: { + const struct btf_array *a = btf_array(t); - case BTF_KIND_ARRAY: { - const struct btf_array *a = btf_array(t); + d->type_states[a->index_type].referenced = 1; + d->type_states[a->type].referenced = 1; + break; + } + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: { + const struct btf_member *m = btf_members(t); - d->type_states[a->index_type].referenced = 1; - d->type_states[a->type].referenced = 1; - break; - } - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); + for (j = 0; j < vlen; j++, m++) + d->type_states[m->type].referenced = 1; + break; + } + case BTF_KIND_FUNC_PROTO: { + const struct btf_param *p = btf_params(t); - for (j = 0; j < vlen; j++, m++) - d->type_states[m->type].referenced = 1; - break; - } - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *p = btf_params(t); + for (j = 0; j < vlen; j++, p++) + d->type_states[p->type].referenced = 1; + break; + } + case BTF_KIND_DATASEC: { + const struct btf_var_secinfo *v = btf_var_secinfos(t); - for (j = 0; j < vlen; j++, p++) - d->type_states[p->type].referenced = 1; - break; - } - case BTF_KIND_DATASEC: { - const struct btf_var_secinfo *v = btf_var_secinfos(t); - - for (j = 0; j < vlen; j++, v++) - d->type_states[v->type].referenced = 1; - break; - } - default: - return -EINVAL; - } + for (j = 0; j < vlen; j++, v++) + d->type_states[v->type].referenced = 1; + break; + } + default: + return -EINVAL; } return 0; } @@ -609,6 +660,9 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) case BTF_KIND_VAR: case BTF_KIND_DATASEC: case BTF_KIND_DECL_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: d->type_states[id].order_state = ORDERED; return 0; @@ -964,6 +1018,44 @@ static void btf_dump_emit_struct_fwd(struct btf_dump *d, __u32 id, btf_dump_type_name(d, id)); } +static void btf_dump_emit_decl_tag(struct btf_dump *d, const struct btf_type *t) +{ + const char *name = btf_name_of(d, t->name_off); + + if (btf_kflag(t)) + btf_dump_printf(d, " __attribute__((%s))", name); + else + btf_dump_printf(d, " __attribute__((btf_decl_tag(\"%s\")))", name); +} + +/* + * btf_dump_resize() keeps d->decl_tags sorted by (target ID, tag ID), so the + * tags of one type form a run that binary search finds the start of, in a + * fixed order so that the same BTF always renders the same C. + */ +static void btf_dump_emit_decl_tags(struct btf_dump *d, __u32 id, int comp_idx) +{ + size_t i, lo = 0, hi = d->decl_tag_cnt, mid; + const struct btf_type *t; + + /* lower bound on target_id */ + while (lo < hi) { + mid = lo + (hi - lo) / 2; + + if (d->decl_tags[mid].target_id < id) + lo = mid + 1; + else + hi = mid; + } + + for (i = lo; i < d->decl_tag_cnt && d->decl_tags[i].target_id == id; i++) { + t = btf__type_by_id(d->btf, d->decl_tags[i].tag_id); + + if (btf_decl_tag(t)->component_idx == comp_idx) + btf_dump_emit_decl_tag(d, t); + } +} + static void btf_dump_emit_struct_def(struct btf_dump *d, __u32 id, const struct btf_type *t, @@ -1009,6 +1101,8 @@ static void btf_dump_emit_struct_def(struct btf_dump *d, prev_bitfield = false; } + /* after the bit-field width */ + btf_dump_emit_decl_tags(d, id, i); btf_dump_printf(d, ";"); } @@ -1028,6 +1122,7 @@ static void btf_dump_emit_struct_def(struct btf_dump *d, } if (packed) btf_dump_printf(d, " __attribute__((packed))"); + btf_dump_emit_decl_tags(d, id, -1); } static const char *missing_base_types[][2] = { @@ -1205,6 +1300,7 @@ static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id, btf_dump_printf(d, "typedef "); btf_dump_emit_type_decl(d, t->type, name, lvl); + btf_dump_emit_decl_tags(d, id, -1); } static int btf_dump_push_decl_stack_id(struct btf_dump *d, __u32 id) @@ -1499,6 +1595,9 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, name = btf_name_of(d, t->name_off); if (btf_kflag(t)) btf_dump_printf(d, " __attribute__((%s))", name); + else if (strcmp(name, "arena") == 0) + /* the tag denotes a pointer into the BPF arena address space */ + btf_dump_printf(d, " __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\")))"); else btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name); break; @@ -2525,6 +2624,9 @@ static int btf_dump_dump_type_data(struct btf_dump *d, case BTF_KIND_FUNC: case BTF_KIND_FUNC_PROTO: case BTF_KIND_DECL_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: err = btf_dump_unsupported_data(d, t, id); break; case BTF_KIND_INT:
diff --git a/tools/lib/bpf/btf_iter.c b/tools/lib/bpf/btf_iter.c index 9a6c822..199063b 100644 --- a/tools/lib/bpf/btf_iter.c +++ b/tools/lib/bpf/btf_iter.c
@@ -29,6 +29,7 @@ int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, case BTF_KIND_FLOAT: case BTF_KIND_ENUM: case BTF_KIND_ENUM64: + case BTF_KIND_LOC_PARAM: it->desc = (struct btf_field_desc) {}; break; case BTF_KIND_FWD: @@ -71,6 +72,20 @@ int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, 1, {offsetof(struct btf_var_secinfo, type)} }; break; + case BTF_KIND_LOC_PROTO: + it->desc = (struct btf_field_desc) { + 0, {}, + sizeof(__u32), + 1, {0}}; + break; + case BTF_KIND_LOCSEC: + it->desc = (struct btf_field_desc) { + 0, {}, + sizeof(struct btf_loc), + 2, {offsetof(struct btf_loc, func), + offsetof(struct btf_loc, loc_proto)}}; + break; + default: return -EINVAL; } @@ -94,6 +109,9 @@ int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, case BTF_KIND_DECL_TAG: case BTF_KIND_TYPE_TAG: case BTF_KIND_DATASEC: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: it->desc = (struct btf_field_desc) { 1, {offsetof(struct btf_type, name_off)} };
diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c index df5fa4b..e55dd75 100644 --- a/tools/lib/bpf/btf_relocate.c +++ b/tools/lib/bpf/btf_relocate.c
@@ -441,7 +441,8 @@ static int btf_relocate_rewrite_strs(struct btf_relocate *r, __u32 i) /* If successful, output of relocation is updated BTF with base BTF pointing * at base_btf, and type ids, strings adjusted accordingly. */ -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map) +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map, + __u32 **str_map) { unsigned int nr_types = btf__type_cnt(btf); const struct btf_header *dist_base_hdr; @@ -512,6 +513,10 @@ int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map) *id_map = r.id_map; r.id_map = NULL; } + if (str_map) { + *str_map = r.str_map; + r.str_map = NULL; + } err_out: free(r.id_map); free(r.str_map);
diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index fe136d0..700c391 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c
@@ -115,7 +115,7 @@ static int elf_sym_iter_new(struct elf_sym_iter *iter, return -ENOENT; } - if (!gelf_getshdr(scn, &sh)) + if (!gelf_getshdr(scn, &sh) || !sh.sh_entsize) return -EINVAL; iter->strtabidx = sh.sh_link;
diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index 9f58998..adb5d2e 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c
@@ -77,6 +77,82 @@ static int probe_kern_global_data(int token_fd) return probe_fd(ret); } +/* + * A continuous fd_array binds every map in it to the program, whether an + * insn refers to the map or not. Kernels without the attribute reject a + * non-zero fd_array_cnt. + */ +static int probe_kern_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) { + ret = -errno; + pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", + __func__, errstr(ret)); + return ret; + } + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + +/* + * The kernel checks an exclusive map bound through fd_array_cnt against the + * digest of the final program. Earlier kernels compared it as the map was + * added, against a digest they had not computed yet, and refused the load for + * the very program the map is exclusive to; kernels without exclusive maps + * refuse the map. + */ +static int probe_kern_excl_map_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + __u8 hash[SHA256_DIGEST_LENGTH]; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash), + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + libbpf_sha256(insns, sizeof(insns), hash); + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_excl", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) + return 0; + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + static int probe_kern_btf(int token_fd) { static const char strs[] = "\0int"; @@ -742,6 +818,12 @@ static struct kern_feature_desc { [FEAT_PERCPU_DATA] = { "kernel supports percpu data", probe_kern_percpu_data, }, + [FEAT_FD_ARRAY_CNT] = { + "continuous fd_array (fd_array_cnt)", probe_kern_fd_array_cnt, + }, + [FEAT_EXCL_MAP_FD_ARRAY_CNT] = { + "exclusive map bound through fd_array_cnt", probe_kern_excl_map_fd_array_cnt, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id)
diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index af3a04f..251392a 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c
@@ -112,13 +112,23 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps) { size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; int i; gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); gen->log_level = log_level; + gen->nr_obj_maps = nr_maps; + gen->max_func_ptr_maps = max_func_ptr_maps; + if (nr_maps + max_func_ptr_maps > MAX_USED_MAPS) { + pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); + gen->error = -E2BIG; + return; + } + /* their fds are closed like the fds of the maps of the object when loading fails */ + nr_maps += max_func_ptr_maps; /* save ctx pointer into R6 */ emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); @@ -385,6 +395,9 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) return gen->error; } emit_sys_close_stack(gen, stack_off(btf_fd)); + /* programs hold their maps with pointers to functions, nothing else needs them */ + for (i = 0; i < gen->nr_func_ptr_maps; i++) + emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->nr_obj_maps + i)); for (i = 0; i < gen->nr_progs; i++) move_stack2ctx(gen, sizeof(struct bpf_loader_ctx) + @@ -1127,52 +1140,41 @@ void bpf_gen__prog_load(struct bpf_gen *gen, gen->nr_progs++; } -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size, __u64 flags) +/* + * if (map_desc[map_idx].initial_value) { + * if (ctx->flags & BPF_SKEL_KERNEL) + * bpf_probe_read_kernel(value, value_size, initial_value); + * else + * bpf_copy_from_user(value, value_size, initial_value); + * nr_more_insns that the caller emits + * } + */ +static void emit_copy_initial_value(struct bpf_gen *gen, int map_idx, int value, + __u32 value_size, int nr_more_insns) { - int attr_size = offsetofend(union bpf_attr, flags); - int map_update_attr, value, key; - union bpf_attr attr; - int zero = 0; + emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, + sizeof(struct bpf_loader_ctx) + + sizeof(struct bpf_map_desc) * map_idx + + offsetof(struct bpf_map_desc, initial_value))); + emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8 + nr_more_insns)); + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value)); + emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, + offsetof(struct bpf_loader_ctx, flags))); + emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); +} - memset(&attr, 0, attr_size); - attr.flags = tgt_endian(flags); +/* Update the element of the map whose fd is in the slot map_idx of fd_array */ +static void emit_map_update_elem(struct bpf_gen *gen, int map_idx, union bpf_attr *attr, + int attr_size, int key, int value, __u32 value_size) +{ + int map_update_attr; - value = add_data(gen, pvalue, value_size); - key = add_data(gen, &zero, sizeof(zero)); - - /* - * if (map_desc[map_idx].initial_value) { - * if (ctx->flags & BPF_SKEL_KERNEL) - * bpf_probe_read_kernel(value, value_size, initial_value); - * else - * bpf_copy_from_user(value, value_size, initial_value); - * } - * - * The runtime initial_value comes from the host-supplied loader - * ctx and would overwrite the blob value that the program signature - * covers and the kernel verifies at load time. For a signed loader - * (gen_hash) the attested blob value must be authoritative, so skip - * the override and leave the signed value in place. - */ - if (!OPTS_GET(gen->opts, gen_hash, false)) { - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); - } - - map_update_attr = add_data(gen, &attr, attr_size); + map_update_attr = add_data(gen, attr, attr_size); pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, @@ -1185,6 +1187,33 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, emit_check_err(gen); } +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, + __u32 value_size, __u64 flags) +{ + int attr_size = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + int value, key; + int zero = 0; + + memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); + + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* + * The runtime initial_value comes from the host-supplied loader + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. + */ + if (!OPTS_GET(gen->opts, gen_hash, false)) + emit_copy_initial_value(gen, map_idx, value, value_size, 0); + + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); +} + void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, int inner_map_idx) { @@ -1214,6 +1243,77 @@ void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slo emit_check_err(gen); } +/* + * A copy of the read-only data map obj_map_idx for a program, with the offsets + * of its functions in it, see create_func_ptr_map() in libbpf.c. It's not + * a map of the object: it's not in the loader ctx. Its content is the content + * of obj_map_idx, that the host may supply when the skeleton is loaded, with + * ptr_cnt 64-bit ptr_vals at ptr_offs. + * Return the index of the map in fd_array for instructions to refer to. + */ +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *pvalue, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt) +{ + int attr_size = offsetofend(union bpf_attr, map_extra); + int map_create_attr, map_idx, key, value, zero = 0, i; + union bpf_attr attr; + + if (gen->nr_func_ptr_maps == gen->max_func_ptr_maps) { + gen->error = -EDOM; /* internal bug */ + return 0; + } + map_idx = gen->nr_obj_maps + gen->nr_func_ptr_maps++; + + memset(&attr, 0, attr_size); + attr.map_type = tgt_endian(BPF_MAP_TYPE_ARRAY); + attr.key_size = tgt_endian((__u32)sizeof(int)); + attr.value_size = tgt_endian(value_size); + attr.max_entries = tgt_endian((__u32)1); + attr.map_flags = tgt_endian((__u32)BPF_F_RDONLY_PROG); + if (map_name) + libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); + + map_create_attr = add_data(gen, &attr, attr_size); + pr_debug("gen: func_ptr_map_create: %s idx %d value_size %u, attr: off %d size %d\n", + map_name, map_idx, value_size, map_create_attr, attr_size); + emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); + debug_ret(gen, "func_ptr_map_create %s idx %d value_size %d", map_name, map_idx, + value_size); + emit_check_err(gen); + /* remember map_fd in fd_array */ + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, blob_fd_array_off(gen, map_idx))); + emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); + + /* pvalue has the pointers already */ + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* see bpf_gen__map_update_elem() */ + if (!OPTS_GET(gen->opts, gen_hash, false)) { + /* the jump over these instructions has 16-bit offset */ + if (ptr_cnt > 10000) { + gen->error = -E2BIG; + return 0; + } + emit_copy_initial_value(gen, obj_map_idx, value, value_size, 3 * ptr_cnt); + /* the content that the host supplied doesn't have them */ + for (i = 0; i < ptr_cnt; i++) { + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value + ptr_offs[i])); + emit(gen, BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, ptr_vals[i])); + } + } + + attr_size = offsetofend(union bpf_attr, flags); + memset(&attr, 0, attr_size); + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); + + bpf_gen__map_freeze(gen, map_idx); + return map_idx; +} + void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) { int attr_size = offsetofend(union bpf_attr, map_fd);
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index bfa64ae..8fcb8cf 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c
@@ -497,6 +497,8 @@ struct bpf_program { bool autoload; bool autoattach; bool sym_global; + /* the program or a function that it calls has callx */ + bool has_callx; bool mark_btf_static; enum bpf_prog_type type; enum bpf_attach_type expected_attach_type; @@ -544,11 +546,32 @@ struct bpf_struct_ops { #define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" +#define DATA_REL_RO_SEC ".data.rel.ro" #define KCONFIG_SEC ".kconfig" #define KSYMS_SEC ".ksyms" #define STRUCT_OPS_SEC ".struct_ops" #define STRUCT_OPS_LINK_SEC ".struct_ops.link" #define ARENA_SEC ".addr_space.1" +/* + * An object with this section keeps its global data in arena instead of + * array maps. What is in the section doesn't matter. .data, .bss and .rodata + * sections become a part of the arena map of the object, like __arena global + * variables are. If the object doesn't declare an arena map libbpf creates + * one that is just large enough for the data. bpf_object__find_map_by_name() + * finds it by the name "arena" and bpf_map__set_max_entries() changes its + * size before the object is loaded. + * Read-only sections with pointers to functions and sections with constant + * strings stay frozen array maps. A constant that a helper or a kfunc takes + * by pointer has to be in such section. + * Pointers to data that are stored in data become addresses of arena. + * The load fails if such pointer points to a section that is not in arena, + * or if the arena is pinned or reused. + * + * It's for programs written in Rust. There are no address spaces in Rust, + * the program accesses arena through plain numbers, so programs of the object + * are loaded with BPF_F_ARENA_SCALAR. + */ +#define ARENA_DATA_SEC ".arena.data" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -601,6 +624,16 @@ struct bpf_map { bool autoattach; __u64 map_extra; struct bpf_program *excl_prog; + /* pointers to functions in the data of an internal map, see obj->func_ptrs */ + struct func_ptr *func_ptrs; + size_t func_ptr_cnt; + /* + * Data of the internal map is a part of arena data at arena_off. + * The map is not created, unless it's a copy that is in arena. + */ + bool in_arena; + bool arena_copy; + size_t arena_off; }; enum extern_type { @@ -751,6 +784,8 @@ struct bpf_object { bool btf_modules_loaded; size_t btf_module_cnt; size_t btf_module_cap; + char **btf_module_allowlist; + ssize_t btf_module_allowlist_cnt; /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ char *log_buf; @@ -759,7 +794,13 @@ struct bpf_object { int *fd_array; size_t fd_array_cap; - size_t fd_array_cnt; + /* + * Slots handed out so far, slot 0 included. A kfunc insn names a module + * BTF slot through its offset, slot 0 stands for vmlinux BTF and stays + * empty, so the array goes to the kernel without a count, unless + * bpf_object__bind_arena() puts the arena of the object there. + */ + size_t fd_array_used; struct usdt_manager *usdt_man; @@ -767,6 +808,17 @@ struct bpf_object { void *arena_data; size_t arena_data_sz; size_t arena_data_off; + bool data_in_arena; + bool arena_mapped; + /* pointers to data in the data of internal maps, when data is in arena */ + struct data_ptr { + int sec_idx; + size_t sec_off; + int targ_sec_idx; + size_t targ_off; + char *sym_name; + } *data_ptrs; + size_t data_ptr_cnt; void *jumptables_data; size_t jumptables_data_sz; @@ -778,6 +830,29 @@ struct bpf_object { } *jumptable_maps; size_t jumptable_map_cnt; + /* + * Pointers to functions found in read-only data sections: tables of + * functions, structures of operations, vtables. Sorted by section + * and offset. + */ + struct func_ptr { + int sec_idx; /* ELF section that contains the pointer */ + size_t sec_off; /* offset of the pointer in the section */ + size_t text_off; /* offset of the function in .text section */ + } *func_ptrs; + size_t func_ptr_cnt; + + /* + * Read-only data with pointers to functions is different for every + * program that uses it, because so are the offsets of the functions. + */ + struct { + struct bpf_program *prog; + int map_idx; + int fd; + } *func_ptr_maps; + size_t func_ptr_map_cnt; + struct kern_feature_cache *feat_cache; char *token_path; int token_fd; @@ -843,6 +918,16 @@ static bool insn_is_pseudo_func(struct bpf_insn *insn) return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } +static bool prog_has_callx(const struct bpf_program *prog) +{ + size_t i; + + for (i = 0; i < prog->insns_cnt; i++) + if (prog->insns[i].code == (BPF_JMP | BPF_CALL | BPF_X)) + return true; + return false; +} + static int bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, const char *name, size_t sec_idx, const char *sec_name, @@ -1223,6 +1308,12 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map) const char *mname; mname = btf__name_by_offset(btf, member->name_off); + if (btf_member_bitfield_size(type, i)) { + pr_warn("struct_ops init_kern %s: local bitfield %s is not supported\n", + map->name, mname); + return -ENOTSUP; + } + moff = member->offset / 8; mdata = data + moff; msize = btf__resolve_size(btf, member->type); @@ -1259,8 +1350,7 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map) } kern_member_idx = kern_member - btf_members(kern_type); - if (btf_member_bitfield_size(type, i) || - btf_member_bitfield_size(kern_type, kern_member_idx)) { + if (btf_member_bitfield_size(kern_type, kern_member_idx)) { pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", map->name, mname); return -ENOTSUP; @@ -1554,6 +1644,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; + obj->efile.arena_data_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -2478,6 +2569,9 @@ static const char *__btf_kind_str(__u16 kind) case BTF_KIND_DECL_TAG: return "decl_tag"; case BTF_KIND_TYPE_TAG: return "type_tag"; case BTF_KIND_ENUM64: return "enum64"; + case BTF_KIND_LOC_PARAM: return "loc_param"; + case BTF_KIND_LOC_PROTO: return "loc_proto"; + case BTF_KIND_LOCSEC: return "locsec"; default: return "unknown"; } } @@ -3018,20 +3112,8 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, } static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, - const char *sec_name, int sec_idx, void *data, size_t data_sz) { - const long page_sz = sysconf(_SC_PAGE_SIZE); - const size_t data_alloc_sz = roundup(data_sz, page_sz); - size_t mmap_sz; - - mmap_sz = bpf_map_mmap_sz(map); - if (data_alloc_sz > mmap_sz) { - pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", - sec_name, mmap_sz, data_sz); - return -E2BIG; - } - obj->arena_data = malloc(data_sz); if (!obj->arena_data) return -ENOMEM; @@ -3044,6 +3126,136 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, return 0; } +static bool map_is_const_str(const struct bpf_map *map) +{ + return map->libbpf_type == LIBBPF_MAP_RODATA && + str_has_pfx(map->real_name, RODATA_SEC ".str"); +} + +static bool map_data_goes_to_arena(const struct bpf_object *obj, const struct bpf_map *map) +{ + int i; + + switch (map->libbpf_type) { + case LIBBPF_MAP_DATA: + case LIBBPF_MAP_BSS: + return true; + case LIBBPF_MAP_RODATA: + break; + default: + return false; + } + + /* + * Const strings that kfuncs and helpers take have to be in a frozen + * map. Pointers to them that are stored in data point to a copy of + * the strings that is in arena. + */ + if (map_is_const_str(map)) + return true; + /* + * Read-only data with pointers stays a frozen map: the kernel + * recognizes pointers to functions in it. + */ + if (str_has_pfx(map->real_name, DATA_REL_RO_SEC)) + return false; + for (i = 0; i < obj->efile.sec_cnt; i++) { + const struct elf_sec_desc *sec = &obj->efile.secs[i]; + + if (sec->sec_type == SEC_RELO && sec->shdr->sh_info == map->sec_idx) + return false; + } + return true; +} + +/* Returns the offset of the data of the map in arena data that is sz bytes so far */ +static size_t map_arena_off(const struct bpf_object *obj, const struct bpf_map *map, size_t sz) +{ + size_t align = obj->efile.secs[map->sec_idx].shdr->sh_addralign; + + return roundup(sz, max(align, sizeof(__u64))); +} + +/* + * Make .data, .bss and .rodata a part of arena data. They follow __arena + * variables of the object, if there are any. + */ +static int bpf_object__init_arena_data(struct bpf_object *obj) +{ + const size_t page_sz = sysconf(_SC_PAGE_SIZE); + size_t sz = obj->arena_data_sz; + struct bpf_map *map; + void *data; + int i; + + if (!obj->data_in_arena) + return 0; + + for (i = 0; i < obj->nr_maps; i++) { + map = &obj->maps[i]; + if (map_data_goes_to_arena(obj, map)) + sz = map_arena_off(obj, map, sz) + map->def.value_size; + } + if (sz == obj->arena_data_sz) + return 0; + + if (obj->arena_map_idx < 0) { + /* bpf_object__init_user_btf_maps() didn't look: there is no .maps section */ + if (obj->efile.arena_data) { + pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", + ARENA_SEC); + return -ENOENT; + } + + map = bpf_object__add_map(obj); + if (IS_ERR(map)) + return PTR_ERR(map); + + map->real_name = strdup("arena"); + map->name = strdup("arena"); + if (!map->real_name || !map->name) { + zfree(&map->real_name); + zfree(&map->name); + return -ENOMEM; + } + map->sec_idx = -1; + map->def.type = BPF_MAP_TYPE_ARENA; + map->def.max_entries = roundup(sz, page_sz) / page_sz; + map->def.map_flags = BPF_F_MMAPABLE; + obj->arena_map_idx = map - obj->maps; + } + + data = realloc(obj->arena_data, sz); + if (!data) + return -ENOMEM; + memset(data + obj->arena_data_sz, 0, sz - obj->arena_data_sz); + sz = obj->arena_data_sz; + obj->arena_data = data; + obj->maps[obj->arena_map_idx].mmaped = data; + + for (i = 0; i < obj->nr_maps; i++) { + map = &obj->maps[i]; + if (!map_data_goes_to_arena(obj, map)) + continue; + sz = map_arena_off(obj, map, sz); + map->arena_off = sz; + memcpy(data + sz, map->mmaped, map->def.value_size); + sz += map->def.value_size; + pr_debug("map '%s': data is in arena at offset %zu\n", map->name, map->arena_off); + if (map_is_const_str(map)) { + map->arena_copy = true; + continue; + } + munmap(map->mmaped, bpf_map_mmap_sz(map)); + /* make bpf_map__initial_value() and bpf_map__set_initial_value() work */ + map->mmaped = data + map->arena_off; + map->autocreate = false; + map->in_arena = true; + } + obj->arena_data_sz = sz; + return 0; +} + static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, const char *pin_root_path) { @@ -3107,8 +3319,7 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, obj->arena_map_idx = i; if (obj->efile.arena_data) { - err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx, - obj->efile.arena_data->d_buf, + err = init_arena_map_data(obj, map, obj->efile.arena_data->d_buf, obj->efile.arena_data->d_size); if (err) return err; @@ -3137,6 +3348,7 @@ static int bpf_object__init_maps(struct bpf_object *obj, err = err ?: bpf_object__init_global_data_maps(obj); err = err ?: bpf_object__init_kconfig_map(obj); err = err ?: bpf_object_init_struct_ops(obj); + err = err ?: bpf_object__init_arena_data(obj); return err; } @@ -3951,6 +4163,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!sh) return -LIBBPF_ERRNO__FORMAT; + /* the second pass has to know it when it sees relocations of data */ + name = elf_sec_str(obj, sh->sh_name); + if (name && strcmp(name, ARENA_DATA_SEC) == 0) + obj->data_in_arena = true; + if (sh->sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warn("elf: multiple symbol tables in %s\n", obj->path); @@ -4030,6 +4247,17 @@ static int bpf_object__elf_collect(struct bpf_object *obj) err = bpf_object__add_programs(obj, data, name, idx); if (err) return err; + } else if (strcmp(name, DATA_REL_RO_SEC) == 0 || + str_has_pfx(name, DATA_REL_RO_SEC ".")) { + /* + * Constants with pointers in them, e.g. vtables, + * that position independent code keeps here to + * have them relocated. There is nothing that + * writes to it after that. + */ + sec_desc->sec_type = SEC_RODATA; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, DATA_SEC) == 0 || str_has_pfx(name, DATA_SEC ".")) { sec_desc->sec_type = SEC_DATA; @@ -4056,6 +4284,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; + } else if (strcmp(name, ARENA_DATA_SEC) == 0) { + /* the marker, see the first pass */ } else if (strcmp(name, JUMPTABLES_SEC) == 0) { obj->jumptables_data = malloc(data->d_size); if (!obj->jumptables_data) @@ -4074,8 +4304,19 @@ static int bpf_object__elf_collect(struct bpf_object *obj) targ_sec_idx >= obj->efile.sec_cnt) return -LIBBPF_ERRNO__FORMAT; - /* Only do relo for section with exec instructions */ + /* + * Only do relo for section with exec instructions, + * struct_ops, maps, and read-only data that might + * have pointers to functions. + */ if (!section_have_execinstr(obj, targ_sec_idx) && + !(obj->data_in_arena && + (!strcmp(name, ".rel" DATA_SEC) || + str_has_pfx(name, ".rel" DATA_SEC "."))) && + strcmp(name, ".rel" RODATA_SEC) && + !str_has_pfx(name, ".rel" RODATA_SEC ".") && + strcmp(name, ".rel" DATA_REL_RO_SEC) && + !str_has_pfx(name, ".rel" DATA_REL_RO_SEC ".") && strcmp(name, ".rel" STRUCT_OPS_SEC) && strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && strcmp(name, ".rel?" STRUCT_OPS_SEC) && @@ -4823,6 +5064,10 @@ static int bpf_program__record_reloc(struct bpf_program *prog, reloc_desc->insn_idx = insn_idx; reloc_desc->map_idx = map_idx; reloc_desc->sym_off = sym->st_value; + if (map->in_arena) { + reloc_desc->map_idx = obj->arena_map_idx; + reloc_desc->sym_off += map->arena_off; + } return 0; } @@ -5041,6 +5286,9 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) if (map_is_created(map)) return libbpf_err(-EBUSY); + if (map->in_arena) + return libbpf_err(-EOPNOTSUPP); + map->autocreate = autocreate; return 0; } @@ -5321,6 +5569,45 @@ bpf_object__reuse_map(struct bpf_map *map) return 0; } +static struct bpf_map *bpf_object__sec_map(struct bpf_object *obj, int sec_idx) +{ + int i; + + for (i = 0; i < obj->nr_maps; i++) + if (bpf_map__is_internal(&obj->maps[i]) && obj->maps[i].sec_idx == sec_idx) + return &obj->maps[i]; + return NULL; +} + +static int bpf_object__mmap_arena(struct bpf_object *obj, struct bpf_map *map) +{ + int i, err; + + map->mmaped = mmap((void *)(long)map->map_extra, + bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, + map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, + map->fd, 0); + if (map->mmaped == MAP_FAILED) { + err = -errno; + map->mmaped = NULL; + pr_warn("map '%s': failed to mmap arena: %s\n", + map->name, errstr(err)); + return err; + } + if (obj->arena_data) { + memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, + obj->arena_data_sz); + zfree(&obj->arena_data); + } + for (i = 0; i < obj->nr_maps; i++) { + struct bpf_map *m = &obj->maps[i]; + + if (m->in_arena) + m->mmaped = map->mmaped + obj->arena_data_off + m->arena_off; + } + return 0; +} + static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { @@ -5681,6 +5968,10 @@ bpf_object__create_maps(struct bpf_object *obj) continue; } + /* see bpf_object__relocate_data_ptrs() */ + if (map->def.type == BPF_MAP_TYPE_ARENA && obj->arena_mapped) + continue; + err = map_set_def_max_entries(map); if (err) goto err_out; @@ -5718,22 +6009,9 @@ bpf_object__create_maps(struct bpf_object *obj) if (err < 0) goto err_out; } else if (map->def.type == BPF_MAP_TYPE_ARENA) { - map->mmaped = mmap((void *)(long)map->map_extra, - bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, - map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, - map->fd, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("map '%s': failed to mmap arena: %s\n", - map->name, errstr(err)); + err = bpf_object__mmap_arena(obj, map); + if (err) return err; - } - if (obj->arena_data) { - memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, - obj->arena_data_sz); - zfree(&obj->arena_data); - } } if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { err = init_map_in_map_slots(obj, map); @@ -5851,6 +6129,21 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, return 0; } +static bool is_btf_mod_allowed(const struct bpf_object *obj, const char *name) +{ + ssize_t i; + + if (obj->btf_module_allowlist_cnt < 0) + return true; + + for (i = 0; i < obj->btf_module_allowlist_cnt; i++) { + if (strcmp(obj->btf_module_allowlist[i], name) == 0) + return true; + } + + return false; +} + static int load_module_btfs(struct bpf_object *obj) { struct bpf_btf_info info; @@ -5873,6 +6166,9 @@ static int load_module_btfs(struct bpf_object *obj) if (!kernel_supports(obj, FEAT_MODULE_BTF)) return 0; + if (obj->btf_module_allowlist_cnt == 0) + return 0; + while (true) { err = bpf_btf_get_next_id(id, &id); if (err && errno == ENOENT) @@ -5915,6 +6211,11 @@ static int load_module_btfs(struct bpf_object *obj) continue; } + if (!is_btf_mod_allowed(obj, name)) { + close(fd); + continue; + } + btf = btf_get_from_fd(fd, obj->btf_vmlinux); err = libbpf_get_error(btf); if (err) { @@ -5939,6 +6240,9 @@ static int load_module_btfs(struct bpf_object *obj) break; } obj->btf_module_cnt++; + + if (obj->btf_module_allowlist_cnt == obj->btf_module_cnt) + break; } if (err) { @@ -6174,8 +6478,11 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { - if (rec->insn_off % BPF_INSN_SZ) - return -EINVAL; + if (rec->insn_off % BPF_INSN_SZ) { + err = -EINVAL; + goto out; + } + insn_idx = rec->insn_off / BPF_INSN_SZ; prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); if (!prog) { @@ -6202,8 +6509,10 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) * relocated, so it's enough to just subtract in-section offset */ insn_idx = insn_idx - prog->sec_insn_off; - if (insn_idx >= prog->insns_cnt) - return -EINVAL; + if (insn_idx >= prog->insns_cnt) { + err = -EINVAL; + goto out; + } insn = &prog->insns[insn_idx]; if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) { @@ -6458,6 +6767,161 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc return err; } +/* + * The kernel recognizes a pointer to a function in a frozen read-only map by + * its value: the offset in bytes of the function in the program. It makes + * callx work for tables of functions, structures of operations and vtables, + * where pointers are mixed with other data. Functions have different offsets + * in different programs, so create a copy of the map for the program. + * The kernel replaces the offsets with the addresses of the functions when it + * loads the program, which has to be the only user of the map. + */ +static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, int map_idx) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + struct bpf_map *map = &obj->maps[map_idx]; + __u32 value_size = map->def.value_size; + size_t i, j, cnt, sec_insn_off; + struct func_ptr *ptrs; + int map_fd, err, zero = 0; + __u64 val; + void *data, *tmp; + + for (i = 0; i < obj->func_ptr_map_cnt; i++) + if (obj->func_ptr_maps[i].prog == prog && + obj->func_ptr_maps[i].map_idx == map_idx) + return obj->func_ptr_maps[i].fd; + + /* the map is not created yet, what it's going to have is in mmaped */ + if (!map->mmaped) + return -EINVAL; + + data = malloc(value_size); + if (!data) + return -ENOMEM; + memcpy(data, map->mmaped, value_size); + + ptrs = map->func_ptrs; + cnt = map->func_ptr_cnt; + for (i = 0; i < cnt; i++) { + if (ptrs[i].sec_off + sizeof(val) > value_size) { + err = -LIBBPF_ERRNO__FORMAT; + goto err_free; + } + /* + * Static functions were appended by bpf_object__append_func_ptrs_code(). + * A global function is in the program only if the code refers to it. + */ + sec_insn_off = ptrs[i].text_off / BPF_INSN_SZ; + for (j = 0; j < prog->subprog_cnt; j++) + if (prog->subprogs[j].sec_insn_off == sec_insn_off) + break; + if (j == prog->subprog_cnt) { + pr_debug("prog '%s': map '%s': no function for the pointer at offset %zu, it's NULL\n", + prog->name, map->name, ptrs[i].sec_off); + val = 0; + } else { + val = (__u64)prog->subprogs[j].sub_insn_off * BPF_INSN_SZ; + } + /* light skeleton can be generated for a target of another endianness */ + if (!is_native_endianness(obj)) + val = bswap_64(val); + memcpy(data + ptrs[i].sec_off, &val, sizeof(val)); + } + + /* + * The kernel takes any aligned 64-bit value that is equal to the offset + * of a function for a pointer. Tell when it's going to get it wrong. + */ + for (i = 0, j = 0; i + sizeof(val) <= value_size; i += sizeof(val)) { + __u32 k; + + while (j < cnt && ptrs[j].sec_off < i) + j++; + if (j < cnt && ptrs[j].sec_off == i) + continue; + memcpy(&val, data + i, sizeof(val)); + if (!is_native_endianness(obj)) + val = bswap_64(val); + if (!val || val % BPF_INSN_SZ) + continue; + for (k = 0; k < prog->subprog_cnt; k++) { + if ((__u64)prog->subprogs[k].sub_insn_off * BPF_INSN_SZ != val) + continue; + pr_warn("prog '%s': map '%s': value %llu at offset %zu is the offset of a function, the kernel will treat it as a pointer to it\n", + prog->name, map->name, (unsigned long long)val, i); + break; + } + } + + if (obj->gen_loader) { + __u32 *ptr_offs = calloc(cnt, sizeof(*ptr_offs)); + __u64 *ptr_vals = calloc(cnt, sizeof(*ptr_vals)); + + if (!ptr_offs || !ptr_vals) { + free(ptr_offs); + free(ptr_vals); + err = -ENOMEM; + goto err_free; + } + for (i = 0; i < cnt; i++) { + ptr_offs[i] = ptrs[i].sec_off; + memcpy(&ptr_vals[i], data + ptrs[i].sec_off, sizeof(val)); + if (!is_native_endianness(obj)) + ptr_vals[i] = bswap_64(ptr_vals[i]); + } + /* it's an index in fd_array of the loader, not an fd */ + map_fd = bpf_gen__func_ptr_map_create(obj->gen_loader, map->name, map_idx, data, + value_size, ptr_offs, ptr_vals, cnt); + free(ptr_offs); + free(ptr_vals); + goto done; + } + + opts.token_fd = obj->token_fd; + if (obj->token_fd) + opts.map_flags |= BPF_F_TOKEN_FD; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, map->name, sizeof(int), value_size, 1, &opts); + if (map_fd < 0) { + err = map_fd; + goto err_free; + } + + err = bpf_map_update_elem(map_fd, &zero, data, 0); + if (!err) + err = bpf_map_freeze(map_fd); + if (err) { + err = -errno; + goto err_close; + } +done: + + tmp = libbpf_reallocarray(obj->func_ptr_maps, obj->func_ptr_map_cnt + 1, + sizeof(*obj->func_ptr_maps)); + if (!tmp) { + err = -ENOMEM; + goto err_close; + } + obj->func_ptr_maps = tmp; + obj->func_ptr_maps[obj->func_ptr_map_cnt].prog = prog; + obj->func_ptr_maps[obj->func_ptr_map_cnt].map_idx = map_idx; + obj->func_ptr_maps[obj->func_ptr_map_cnt].fd = map_fd; + obj->func_ptr_map_cnt++; + + pr_debug("prog '%s': created a copy of map '%s' with %zu pointers to functions\n", + prog->name, map->name, cnt); + free(data); + return map_fd; + +err_close: + if (!obj->gen_loader) + close(map_fd); +err_free: + free(data); + return err; +} + /* Relocate data references within program code: * - map references; * - global variable references; @@ -6495,7 +6959,20 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) if (relo->map_idx == obj->arena_map_idx) insn[1].imm += obj->arena_data_off; - if (obj->gen_loader) { + if (map->autocreate && map->func_ptr_cnt && prog->has_callx) { + int map_fd; + + /* the program gets its own map with pointers to its functions */ + map_fd = create_func_ptr_map(obj, prog, relo->map_idx); + if (map_fd < 0) { + pr_warn("prog '%s': relo #%d: can't create a copy of map '%s' with pointers to functions\n", + prog->name, i, map->name); + return map_fd; + } + insn[0].src_reg = obj->gen_loader ? BPF_PSEUDO_MAP_IDX_VALUE : + BPF_PSEUDO_MAP_VALUE; + insn[0].imm = map_fd; + } else if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; insn[0].imm = relo->map_idx; } else if (map->autocreate) { @@ -6931,6 +7408,75 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, return 0; } +/* Append to the main program all functions that the data of the map points to */ +static int +bpf_object__append_func_ptrs_code(struct bpf_object *obj, struct bpf_program *main_prog, + const struct bpf_map *map) +{ + struct bpf_program *subprog; + size_t i, cnt, sec_insn_off; + struct func_ptr *ptrs; + int err; + + ptrs = map->func_ptrs; + cnt = map->func_ptr_cnt; + for (i = 0; i < cnt; i++) { + sec_insn_off = ptrs[i].text_off / BPF_INSN_SZ; + subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sec_insn_off); + if (!subprog || subprog->sec_insn_off != sec_insn_off) { + pr_warn("prog '%s': map '%s': no function at .text+%zu for the pointer at offset %zu\n", + main_prog->name, map->name, ptrs[i].text_off, ptrs[i].sec_off); + return -LIBBPF_ERRNO__RELOC; + } + + /* + * callx can't call global functions. Don't add one to the + * program only because the data points to it. + */ + if (subprog->sym_global) + continue; + + /* see the comment in bpf_object__reloc_code() */ + if (subprog->sub_insn_off == 0) { + err = bpf_object__append_subprog_code(obj, main_prog, subprog); + if (err) + return err; + err = bpf_object__reloc_code(obj, main_prog, subprog); + if (err) + return err; + } + } + return 0; +} + +/* + * The program that has callx might call any function that the data it refers + * to points to. Append them. Programs that don't have callx can't call them + * and the kernel doesn't look for pointers to functions in their data. + */ +static int +bpf_object__append_func_ptrs(struct bpf_object *obj, struct bpf_program *prog) +{ + size_t i; + int err; + + prog->has_callx = prog_has_callx(prog); + if (!prog->has_callx) + return 0; + + /* relocations of the functions that are appended are appended too */ + for (i = 0; i < prog->nr_reloc; i++) { + struct reloc_desc *relo = &prog->reloc_desc[i]; + + if (relo->type != RELO_DATA || !obj->maps[relo->map_idx].func_ptr_cnt) + continue; + err = bpf_object__append_func_ptrs_code(obj, prog, &obj->maps[relo->map_idx]); + if (err) + return err; + } + return 0; +} + /* * Relocate sub-program calls. * @@ -7033,7 +7579,7 @@ bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) if (err) return err; - return 0; + return bpf_object__append_func_ptrs(obj, prog); } static void @@ -7480,6 +8026,81 @@ static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_progra return err; } +/* + * Turn pointers to data that are stored in data into addresses of arena. + * The arena is created here, ahead of the other maps: the address has to be + * known before the maps that hold the pointers are created. + */ +static int bpf_object__relocate_data_ptrs(struct bpf_object *obj) +{ + struct bpf_map *arena, *map, *targ; + __u64 addr, val; + size_t i; + int err; + + if (!obj->data_ptr_cnt) + return 0; + + for (i = 0; i < obj->data_ptr_cnt; i++) { + struct data_ptr *p = &obj->data_ptrs[i]; + + if (p->targ_sec_idx < 0) + continue; + map = bpf_object__sec_map(obj, p->sec_idx); + targ = bpf_object__sec_map(obj, p->targ_sec_idx); + if (map && (map->in_arena || map->autocreate) && + (!targ || (!targ->in_arena && !targ->arena_copy))) { + pr_warn("sec '%s': pointer to '%s' at offset %zu can't be resolved: sec '%s' is not in arena\n", + map->real_name, p->sym_name, p->sec_off, + targ ? targ->real_name : "<?>"); + return -LIBBPF_ERRNO__RELOC; + } + } + + if (obj->gen_loader) { + pr_warn("pointers to data in data are not supported by light skeleton\n"); + return -ENOTSUP; + } + + if (obj->arena_map_idx < 0) + return 0; + + arena = &obj->maps[obj->arena_map_idx]; + if (!arena->autocreate) + return 0; + /* libbpf doesn't mmap an arena that it didn't create, so the address is not known */ + if (arena->reused || arena->pin_path) { + pr_warn("map '%s': pointers to data in data are not supported with pinned or reused arena\n", + arena->name); + return -ENOTSUP; + } + + err = bpf_object__create_map(obj, arena, false); + err = err ?: bpf_object__mmap_arena(obj, arena); + if (err) { + pr_warn("map '%s': failed to create: %s\n", arena->name, errstr(err)); + return err; + } + obj->arena_mapped = true; + + for (i = 0; i < obj->data_ptr_cnt; i++) { + struct data_ptr *p = &obj->data_ptrs[i]; + + map = bpf_object__sec_map(obj, p->sec_idx); + if (!map || (!map->in_arena && !map->autocreate)) + continue; + + addr = (__u64)(unsigned long)arena->mmaped + obj->arena_data_off + p->targ_off; + if (p->targ_sec_idx >= 0) + addr += bpf_object__sec_map(obj, p->targ_sec_idx)->arena_off; + + memcpy(&val, map->mmaped + p->sec_off, sizeof(val)); + val += addr; + memcpy(map->mmaped + p->sec_off, &val, sizeof(val)); + } + return 0; +} + static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) { struct bpf_program *prog; @@ -7496,14 +8117,28 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat bpf_object__sort_relos(obj); } - /* place globals at the end of the arena (if supported) */ - if (obj->arena_map_idx >= 0 && kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) { + if (obj->arena_map_idx >= 0) { struct bpf_map *arena_map = &obj->maps[obj->arena_map_idx]; + size_t data_sz = roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE)); + size_t mmap_sz = bpf_map_mmap_sz(arena_map); - obj->arena_data_off = bpf_map_mmap_sz(arena_map) - - roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE)); + if (data_sz > mmap_sz) { + pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global %s of size %zu\n", + arena_map->name, mmap_sz, + obj->data_in_arena ? "data" : "__arena variables", + obj->arena_data_sz); + return -E2BIG; + } + + /* place globals at the end of the arena (if supported) */ + if (kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) + obj->arena_data_off = mmap_sz - data_sz; } + err = bpf_object__relocate_data_ptrs(obj); + if (err) + return err; + /* Before relocating calls pre-process relocations and mark * few ld_imm64 instructions that points to subprogs. * Otherwise bpf_object__reloc_code() later would have to consider @@ -7736,6 +8371,180 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } +/* + * A pointer to data that is stored in data. When data is in arena + * the program dereferences what it loads from data, so the pointer has + * to be the address of the target in arena. + */ +static int bpf_object__collect_data_ptr(struct bpf_object *obj, const char *relo_sec_name, + int relo_idx, size_t sec_idx, const Elf64_Rel *rel, + const Elf64_Sym *sym) +{ + Elf_Data *scn_data = obj->efile.secs[sec_idx].data; + const char *sym_name = elf_sym_str(obj, sym->st_name) ?: "<?>"; + struct data_ptr *ptrs; + + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_shndx < obj->efile.sec_cnt) + sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)) ?: "<?>"; + + if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || + sym->st_shndx >= obj->efile.sec_cnt || + (sym->st_shndx != obj->efile.arena_data_shndx && + !bpf_object__shndx_is_data(obj, sym->st_shndx)) || + rel->r_offset >= scn_data->d_size || + scn_data->d_size - rel->r_offset < sizeof(__u64)) { + pr_warn("sec '%s': relo #%d: can't resolve pointer to '%s' at offset %zu when data is in arena\n", + relo_sec_name, relo_idx, sym_name, (size_t)rel->r_offset); + return -LIBBPF_ERRNO__RELOC; + } + + ptrs = libbpf_reallocarray(obj->data_ptrs, obj->data_ptr_cnt + 1, sizeof(*ptrs)); + if (!ptrs) + return -ENOMEM; + obj->data_ptrs = ptrs; + + ptrs[obj->data_ptr_cnt].sec_idx = sec_idx; + ptrs[obj->data_ptr_cnt].sec_off = rel->r_offset; + /* __arena variables are at the start of arena data */ + ptrs[obj->data_ptr_cnt].targ_sec_idx = + sym->st_shndx == obj->efile.arena_data_shndx ? -1 : sym->st_shndx; + ptrs[obj->data_ptr_cnt].targ_off = sym->st_value; + ptrs[obj->data_ptr_cnt].sym_name = strdup(sym_name); + if (!ptrs[obj->data_ptr_cnt].sym_name) + return -ENOMEM; + obj->data_ptr_cnt++; + + pr_debug("sec '%s': relo #%d: pointer at offset %zu to '%s'\n", + relo_sec_name, relo_idx, (size_t)rel->r_offset, sym_name); + return 0; +} + +/* Collect pointers in a data section that went to arena */ +static int bpf_object__collect_data_relos(struct bpf_object *obj, + Elf64_Shdr *shdr, Elf_Data *data) +{ + int i, err, nrels = shdr->sh_size / shdr->sh_entsize; + const char *relo_sec_name; + Elf64_Sym *sym; + Elf64_Rel *rel; + + relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: "<?>"; + for (i = 0; i < nrels; i++) { + rel = elf_rel_by_idx(data, i); + sym = rel ? elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)) : NULL; + if (!sym) { + pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); + return -LIBBPF_ERRNO__FORMAT; + } + err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, shdr->sh_info, rel, sym); + if (err) + return err; + } + return 0; +} + +/* + * Collect pointers to functions in a read-only data section. They are + * R_BPF_64_ABS64 relocations against .text section, where the offset of + * a static function in the section is stored in place. Relocations in data + * sections were ignored before pointers to functions were supported. Those + * that are something else, e.g. pointers to data, still are, unless data + * is in arena. + */ +static int bpf_object__collect_rodata_relos(struct bpf_object *obj, + Elf64_Shdr *shdr, Elf_Data *data) +{ + size_t sec_idx = shdr->sh_info, sym_idx; + int i, nrels = shdr->sh_size / shdr->sh_entsize; + const char *relo_sec_name; + struct func_ptr *ptrs; + Elf_Data *scn_data; + Elf64_Sym *sym; + Elf64_Rel *rel; + __u64 addend; + + relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: "<?>"; + scn_data = obj->efile.secs[sec_idx].data; + if (!scn_data) + return -LIBBPF_ERRNO__FORMAT; + + for (i = 0; i < nrels; i++) { + rel = elf_rel_by_idx(data, i); + if (!rel) { + pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); + return -LIBBPF_ERRNO__FORMAT; + } + + sym_idx = ELF64_R_SYM(rel->r_info); + sym = elf_sym_by_idx(obj, sym_idx); + if (!sym) { + pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", + relo_sec_name, sym_idx, i); + return -LIBBPF_ERRNO__FORMAT; + } + + if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || + !sym_is_subprog(sym, obj->efile.text_shndx)) { + int err; + + if (obj->data_in_arena) { + err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, + sec_idx, rel, sym); + if (err) + return err; + continue; + } + pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", + relo_sec_name, i); + continue; + } + + /* the kernel finds aligned pointers only */ + if (rel->r_offset % sizeof(__u64) || rel->r_offset >= scn_data->d_size || + scn_data->d_size - rel->r_offset < sizeof(__u64)) { + pr_debug("sec '%s': relo #%d: unsupported offset 0x%zx, skipping...\n", + relo_sec_name, i, (size_t)rel->r_offset); + continue; + } + + memcpy(&addend, scn_data->d_buf + rel->r_offset, sizeof(addend)); + if (!is_native_endianness(obj)) + addend = bswap_64(addend); + if ((sym->st_value + addend) % BPF_INSN_SZ) { + pr_debug("sec '%s': relo #%d: bad pointer to a function at offset %zu+%llu, skipping...\n", + relo_sec_name, i, (size_t)sym->st_value, + (unsigned long long)addend); + continue; + } + + ptrs = libbpf_reallocarray(obj->func_ptrs, obj->func_ptr_cnt + 1, sizeof(*ptrs)); + if (!ptrs) + return -ENOMEM; + obj->func_ptrs = ptrs; + + ptrs[obj->func_ptr_cnt].sec_idx = sec_idx; + ptrs[obj->func_ptr_cnt].sec_off = rel->r_offset; + ptrs[obj->func_ptr_cnt].text_off = sym->st_value + addend; + obj->func_ptr_cnt++; + + pr_debug("sec '%s': relo #%d: pointer at offset %zu to a function at .text+%zu\n", + relo_sec_name, i, (size_t)rel->r_offset, (size_t)(sym->st_value + addend)); + } + return 0; +} + +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct func_ptr *a = _a; + const struct func_ptr *b = _b; + + if (a->sec_idx != b->sec_idx) + return a->sec_idx < b->sec_idx ? -1 : 1; + if (a->sec_off != b->sec_off) + return a->sec_off < b->sec_off ? -1 : 1; + return 0; +} + static int bpf_object__collect_relos(struct bpf_object *obj) { int i, err; @@ -7758,7 +8567,11 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return -LIBBPF_ERRNO__INTERNAL; } - if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) + if (obj->efile.secs[idx].sec_type == SEC_RODATA) + err = bpf_object__collect_rodata_relos(obj, shdr, data); + else if (obj->efile.secs[idx].sec_type == SEC_DATA) + err = bpf_object__collect_data_relos(obj, shdr, data); + else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) err = bpf_object__collect_map_relos(obj, shdr, data); @@ -7768,6 +8581,25 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return err; } + /* sort by section, so that pointers in the data of a map are next to each other */ + if (obj->func_ptr_cnt) + qsort(obj->func_ptrs, obj->func_ptr_cnt, sizeof(*obj->func_ptrs), cmp_func_ptrs); + + for (i = 0; i < obj->nr_maps; i++) { + struct bpf_map *map = &obj->maps[i]; + size_t j; + + if (map->libbpf_type != LIBBPF_MAP_RODATA) + continue; + for (j = 0; j < obj->func_ptr_cnt; j++) { + if (obj->func_ptrs[j].sec_idx != map->sec_idx) + continue; + if (!map->func_ptr_cnt) + map->func_ptrs = &obj->func_ptrs[j]; + map->func_ptr_cnt++; + } + } + bpf_object__sort_relos(obj); return 0; } @@ -7886,6 +8718,19 @@ static int tracing_multi_mod_fd(struct bpf_program *prog, int *btf_obj_fd) return 0; } +static int libbpf_setup_prog_flags(struct bpf_program *prog, long cookie) +{ + enum sec_def_flags def = cookie; + + if (def & SEC_SLEEPABLE) + prog->prog_flags |= BPF_F_SLEEPABLE; + + if (def & SEC_XDP_FRAGS) + prog->prog_flags |= BPF_F_XDP_HAS_FRAGS; + + return 0; +} + /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ static int libbpf_prepare_prog_load(struct bpf_program *prog, struct bpf_prog_load_opts *opts, long cookie) @@ -7896,12 +8741,6 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) opts->expected_attach_type = 0; - if (def & SEC_SLEEPABLE) - opts->prog_flags |= BPF_F_SLEEPABLE; - - if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) - opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; - /* special check for usdt to use uprobe_multi link */ if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { /* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type @@ -7966,6 +8805,56 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); +/* + * Bind the arena of the object to the program about to be loaded. A program + * has an arena only when an insn refers to the map, but the kernel can hand + * it arena pointers all the same, as struct_ops arguments, through tagged + * struct members or as kfunc return values, and the verifier rejects those + * when the program has no arena to place them in. A continuous fd_array + * binds every map in it, the arena takes slot 0, which is free: a kfunc insn + * with offset 0 names vmlinux BTF rather than a slot. + */ +static int bpf_object__bind_arena(struct bpf_object *obj, struct bpf_program *prog, + struct bpf_prog_load_opts *load_attr) +{ + struct bpf_map *map; + int err; + + if (obj->gen_loader || obj->arena_map_idx < 0 || !kernel_supports(obj, FEAT_FD_ARRAY_CNT)) + return 0; + map = &obj->maps[obj->arena_map_idx]; + /* + * Nothing to bind when the arena is not created, or no longer is after + * a failed load, when it is exclusive to another program, and nothing + * for an offloaded program, which the kernel does not let near an + * arena. + */ + if (!map->autocreate || map->fd < 0 || (map->excl_prog && map->excl_prog != prog) || + load_attr->prog_ifindex) + return 0; + /* + * An arena exclusive to the program itself needs a kernel that checks + * the map against the digest of the final program. Earlier kernels + * compared a map bound through the fd_array against a digest they had + * not computed yet and refused the load; there the program binds its + * arena through its insns, as before. + */ + if (map->excl_prog && !kernel_supports(obj, FEAT_EXCL_MAP_FD_ARRAY_CNT)) + return 0; + + if (!obj->fd_array_used) { + err = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 1); + if (err) + return err; + obj->fd_array_used = 1; + } + obj->fd_array[0] = map->fd; + load_attr->fd_array = obj->fd_array; + /* with slot 0 filled every slot handed out is valid, so the array can go with a count */ + load_attr->fd_array_cnt = obj->fd_array_used; + return 0; +} + static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, const char *license, __u32 kern_version, int *prog_fd) @@ -8025,7 +8914,13 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; + /* the program accesses its data in arena through plain numbers */ + if (obj->data_in_arena) + load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; + err = bpf_object__bind_arena(obj, prog, &load_attr); + if (err) + return err; load_attr.token_fd = obj->token_fd; if (obj->token_fd) @@ -8096,7 +8991,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog for (i = 0; i < obj->nr_maps; i++) { map = &prog->obj->maps[i]; - if (map->libbpf_type != LIBBPF_MAP_RODATA) + if (map->libbpf_type != LIBBPF_MAP_RODATA || map->in_arena) continue; if (bpf_prog_bind_map(ret, map->fd, NULL)) { @@ -8481,6 +9376,8 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, const struct bpf_object_open_opts *opts) { const char *kconfig, *btf_tmp_path, *token_path; + size_t mod_allow_cnt, i, j; + const char **mod_allow; struct bpf_object *obj; int err; char *log_buf; @@ -8525,6 +9422,22 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, if (token_path && strlen(token_path) >= PATH_MAX) return ERR_PTR(-ENAMETOOLONG); + mod_allow = OPTS_GET(opts, btf_module_allowlist, NULL); + mod_allow_cnt = OPTS_GET(opts, btf_module_allowlist_cnt, 0); + + if (mod_allow_cnt > SSIZE_MAX || (!mod_allow && mod_allow_cnt > 0)) + return ERR_PTR(-EINVAL); + + for (i = 0; i < mod_allow_cnt; i++) { + if (!mod_allow[i] || !mod_allow[i][0]) + return ERR_PTR(-EINVAL); + + for (j = 0; j < i; j++) { + if (strcmp(mod_allow[i], mod_allow[j]) == 0) + return ERR_PTR(-EINVAL); + } + } + obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); if (IS_ERR(obj)) return obj; @@ -8563,6 +9476,24 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, } } + obj->btf_module_allowlist_cnt = mod_allow ? mod_allow_cnt : -1; + if (mod_allow_cnt > 0) { + obj->btf_module_allowlist = + calloc(mod_allow_cnt, sizeof(*obj->btf_module_allowlist)); + if (!obj->btf_module_allowlist) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < mod_allow_cnt; i++) { + obj->btf_module_allowlist[i] = strdup(mod_allow[i]); + if (!obj->btf_module_allowlist[i]) { + err = -ENOMEM; + goto out; + } + } + } + err = bpf_object__elf_init(obj); err = err ? : bpf_object__elf_collect(obj); err = err ? : bpf_object__collect_externs(obj); @@ -8844,22 +9775,22 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, /* set index for module BTF fd in fd_array, if unset */ if (mod_btf && !mod_btf->fd_array_idx) { /* insn->off is s16 */ - if (obj->fd_array_cnt == INT16_MAX) { + if (obj->fd_array_used == INT16_MAX) { pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", ext->name, mod_btf->fd_array_idx); return -E2BIG; } /* Cannot use index 0 for module BTF fd */ - if (!obj->fd_array_cnt) - obj->fd_array_cnt = 1; + if (!obj->fd_array_used) + obj->fd_array_used = 1; ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), - obj->fd_array_cnt + 1); + obj->fd_array_used + 1); if (ret) return ret; - mod_btf->fd_array_idx = obj->fd_array_cnt; + mod_btf->fd_array_idx = obj->fd_array_used; /* we assume module BTF FD is always >0 */ - obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; + obj->fd_array[obj->fd_array_used++] = mod_btf->fd; } ext->is_set = true; @@ -9089,8 +10020,10 @@ static void bpf_object_cleanup_btf(struct bpf_object *obj) static void bpf_object_post_load_cleanup(struct bpf_object *obj) { - /* clean up fd_array */ + /* clean up fd_array, and the slots handed out with it */ zfree(&obj->fd_array); + obj->fd_array_cap = 0; + obj->fd_array_used = 0; /* clean up BTF */ bpf_object_cleanup_btf(obj); @@ -9144,7 +10077,27 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch * permit cross-endian creation of "light skeleton". */ if (obj->gen_loader) { - bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); + int nr_func_ptr_maps = 0, nr_progs = 0, i; + bool text_has_callx = false; + + /* + * Every program that has callx may get a copy of every map with + * pointers to functions. Which functions a program calls is not + * known yet. Any of them may call the ones that have callx. + */ + for (i = 0; i < obj->nr_maps; i++) + if (obj->maps[i].autocreate && obj->maps[i].func_ptr_cnt) + nr_func_ptr_maps++; + for (i = 0; i < obj->nr_programs; i++) + if (prog_is_subprog(obj, &obj->programs[i]) && + prog_has_callx(&obj->programs[i])) + text_has_callx = true; + for (i = 0; i < obj->nr_programs; i++) + if (obj->programs[i].autoload && !prog_is_subprog(obj, &obj->programs[i]) && + (text_has_callx || prog_has_callx(&obj->programs[i]))) + nr_progs++; + bpf_gen__init(obj->gen_loader, obj->log_level | extra_log_level, + obj->nr_programs, obj->nr_maps, nr_func_ptr_maps * nr_progs); } else if (!is_native_endianness(obj)) { pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); return libbpf_err(-LIBBPF_ERRNO__ENDIAN); @@ -9604,7 +10557,7 @@ static void bpf_map__destroy(struct bpf_map *map) zfree(&map->init_slots); map->init_slots_sz = 0; - if (map->mmaped && map->mmaped != map->obj->arena_data) + if (map->mmaped && map->mmaped != map->obj->arena_data && !map->in_arena) munmap(map->mmaped, bpf_map_mmap_sz(map)); map->mmaped = NULL; @@ -9684,6 +10637,21 @@ void bpf_object__close(struct bpf_object *obj) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); + for (i = 0; i < obj->func_ptr_map_cnt; i++) + if (!obj->gen_loader) + close(obj->func_ptr_maps[i].fd); + zfree(&obj->func_ptr_maps); + zfree(&obj->func_ptrs); + for (i = 0; i < obj->data_ptr_cnt; i++) + zfree(&obj->data_ptrs[i].sym_name); + zfree(&obj->data_ptrs); + + if (obj->btf_module_allowlist) { + for (i = 0; i < obj->btf_module_allowlist_cnt; i++) + zfree(&obj->btf_module_allowlist[i]); + zfree(&obj->btf_module_allowlist); + } + free(obj); } @@ -9938,6 +10906,16 @@ int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) return 0; } +int bpf_program__add_flags(struct bpf_program *prog, __u32 flags) +{ + return bpf_program__set_flags(prog, prog->prog_flags | flags); +} + +int bpf_program__clear_flags(struct bpf_program *prog, __u32 flags) +{ + return bpf_program__set_flags(prog, prog->prog_flags & ~flags); +} + __u32 bpf_program__log_level(const struct bpf_program *prog) { return prog->log_level; @@ -9960,9 +10938,9 @@ const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_siz int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) { - if (log_size && !log_buf) + if (!!log_buf != !!log_size) return libbpf_err(-EINVAL); - if (prog->log_size > UINT_MAX) + if (log_size > UINT_MAX) return libbpf_err(-EINVAL); if (prog->obj->state >= OBJ_LOADED) return libbpf_err(-EBUSY); @@ -10023,10 +11001,18 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex; attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; - attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt; + attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); + /* the arena of the object goes with the program, as in bpf_object__load() */ + if (!OPTS_GET(opts, fd_array, NULL) && !OPTS_GET(opts, fd_array_cnt, 0)) { + err = bpf_object__bind_arena(obj, prog, &attr); + if (err) + return libbpf_err(err); + } attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; + if (obj->data_in_arena) + attr.prog_flags |= BPF_F_ARENA_SCALAR; prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); if (!prog_btf_fd && obj->btf) @@ -10106,6 +11092,7 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts .prog_type = BPF_PROG_TYPE_##ptype, \ .expected_attach_type = atype, \ .cookie = (long)(flags), \ + .prog_setup_fn = libbpf_setup_prog_flags, \ .prog_prepare_load_fn = libbpf_prepare_prog_load, \ __VA_ARGS__ \ } @@ -11032,6 +12019,9 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) if (map_is_created(map)) return libbpf_err(-EBUSY); + if (map->in_arena) + return libbpf_err(-EOPNOTSUPP); + if (map->mmaped) { size_t mmap_old_sz, mmap_new_sz; int err; @@ -11745,7 +12735,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, errstr(bit)); return bit; } - attr.config |= 1 << bit; + attr.config |= 1ULL << bit; } attr.size = attr_sz; attr.type = type; @@ -14162,6 +15152,70 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) return &link->link; } +struct bpf_link *bpf_map__attach_cgroup_opts(const struct bpf_map *map, int cgroup_fd, + const struct bpf_cgroup_opts *opts) +{ + LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); + struct bpf_link_struct_ops *link; + __u32 relative_id, zero = 0; + int err, fd, relative_fd; + + if (!OPTS_VALID(opts, bpf_cgroup_opts)) + return libbpf_err_ptr(-EINVAL); + + if (!bpf_map__is_struct_ops(map)) { + pr_warn("map '%s': can't attach non-struct_ops map\n", map->name); + return libbpf_err_ptr(-EINVAL); + } + + if (map->fd < 0) { + pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name); + return libbpf_err_ptr(-EINVAL); + } + + if (!(map->def.map_flags & BPF_F_LINK)) { + pr_warn("map '%s': can't attach to cgroup without BPF_F_LINK\n", map->name); + return libbpf_err_ptr(-EINVAL); + } + + relative_id = OPTS_GET(opts, relative_id, 0); + relative_fd = OPTS_GET(opts, relative_fd, 0); + + if (relative_fd && relative_id) { + pr_warn("map '%s': relative_fd and relative_id cannot be set at the same time\n", + map->name); + return libbpf_err_ptr(-EINVAL); + } + + link_create_opts.cgroup.expected_revision = OPTS_GET(opts, expected_revision, 0); + link_create_opts.cgroup.relative_fd = relative_fd; + link_create_opts.cgroup.relative_id = relative_id; + link_create_opts.flags = OPTS_GET(opts, flags, 0); + + link = calloc(1, sizeof(*link)); + if (!link) + return libbpf_err_ptr(-ENOMEM); + + err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); + if (err && err != -EBUSY) { + free(link); + return libbpf_err_ptr(err); + } + + link->link.detach = bpf_link__detach_struct_ops; + + fd = bpf_link_create(map->fd, cgroup_fd, BPF_STRUCT_OPS, &link_create_opts); + if (fd < 0) { + free(link); + return libbpf_err_ptr(fd); + } + + link->link.fd = fd; + link->map_fd = map->fd; + + return &link->link; +} + /* * Swap the back struct_ops of a link with a new struct_ops map. */
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index b965ad5..03f794f 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h
@@ -224,10 +224,24 @@ struct bpf_object_open_opts { * point (/sys/fs/bpf), in case this default behavior is undesirable. */ const char *bpf_token_path; + /* + * Optional allowlist of kernel module names whose BTFs libbpf is + * allowed to load. The allowlist limits which kernel module BTFs libbpf + * will consult wherever module BTF might be needed. + * + * When the option is not specified, the existing behavior remains + * unchanged. An explicitly specified empty list prevents libbpf from + * consulting any kernel module BTFs. + * + * The list must contain valid, non-empty module names and must not + * contain duplicate entries; otherwise -EINVAL is returned. + */ + const char **btf_module_allowlist; + size_t btf_module_allowlist_cnt; size_t :0; }; -#define bpf_object_open_opts__last_field bpf_token_path +#define bpf_object_open_opts__last_field btf_module_allowlist_cnt /** * @brief **bpf_object__open()** creates a bpf_object by opening @@ -960,6 +974,9 @@ bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd, struct bpf_map; LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map); +LIBBPF_API struct bpf_link *bpf_map__attach_cgroup_opts(const struct bpf_map *map, + int cgroup_fd, + const struct bpf_cgroup_opts *opts); LIBBPF_API int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map); struct bpf_iter_attach_opts { @@ -1011,6 +1028,28 @@ bpf_program__set_expected_attach_type(struct bpf_program *prog, LIBBPF_API __u32 bpf_program__flags(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_flags(struct bpf_program *prog, __u32 flags); +/** + * @brief **bpf_program__add_flags()** adds one or more flags to the BPF + * program, preserving any existing flags. + * + * @param prog BPF program + * @param flags the flags to add + * + * @return 0, on success; negative error code, otherwise + */ +LIBBPF_API int bpf_program__add_flags(struct bpf_program *prog, __u32 flags); + +/** + * @brief **bpf_program__clear_flags()** clears one or more flags from the BPF + * program, preserving any other flags not explicitly cleared. + * + * @param prog BPF program + * @param flags the flags to clear + * + * @return 0, on success; negative error code, otherwise + */ +LIBBPF_API int bpf_program__clear_flags(struct bpf_program *prog, __u32 flags); + /* Per-program log level and log buffer getters/setters. * See bpf_object_open_opts comments regarding log_level and log_buf * interactions.
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 08ab2ea..eeba407 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map
@@ -458,8 +458,21 @@ LIBBPF_1.8.0 { global: + bpf_map__attach_cgroup_opts; + bpf_prog_stream_open; + bpf_program__add_flags; bpf_program__attach_tracing_multi; + bpf_program__clear_flags; bpf_program__clone; + btf__add_loc_param; + btf__add_loc_param_value; + btf__add_loc_proto; + btf__add_loc_proto_param; + btf__add_locsec; + btf__add_locsec_loc; btf__find_by_name_kind_own; btf__new_empty_opts; } LIBBPF_1.7.0; + +LIBBPF_1.9.0 { +} LIBBPF_1.8.0;
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4c46d34..e90b533 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h
@@ -77,7 +77,7 @@ #define JUMPTABLES_SEC ".jumptables" #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) @@ -249,7 +249,8 @@ const char *btf_kind_str(const struct btf_type *t); const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); const struct btf_header *btf_header(const struct btf *btf); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map, + __u32 **str_map); bool btf_type_is_traceable_func(const struct btf *btf, const struct btf_type *t); static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) @@ -259,7 +260,7 @@ static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) static inline __u32 btf_type_info(int kind, int vlen, int kflag) { - return (kflag << 31) | (kind << 24) | vlen; + return ((__u32)kflag << 31) | (kind << 24) | vlen; } enum map_def_parts { @@ -403,6 +404,10 @@ enum kern_feature_id { FEAT_BPF_SYSCALL_COMMON_ATTRS, /* Kernel supports percpu data */ FEAT_PERCPU_DATA, + /* Kernel binds every map in a continuous fd_array (fd_array_cnt) to the program */ + FEAT_FD_ARRAY_CNT, + /* Kernel checks an exclusive map bound through fd_array_cnt against the final digest */ + FEAT_EXCL_MAP_FD_ARRAY_CNT, __FEAT_CNT, }; @@ -580,7 +585,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter {
diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index e408194..5acf0f8 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c
@@ -62,7 +62,7 @@ static __u32 get_ubuntu_kernel_version(void) */ static __u32 get_debian_kernel_version(struct utsname *info) { - __u32 major, minor, patch; + __u32 major, minor, patch = 0; char *p; p = strstr(info->version, "Debian "); @@ -71,10 +71,13 @@ static __u32 get_debian_kernel_version(struct utsname *info) return 0; } - if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) != 3) - return 0; + if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) == 3) + return KERNEL_VERSION(major, minor, patch); - return KERNEL_VERSION(major, minor, patch); + if (sscanf(p, "Debian %u.%u", &major, &minor) == 2) + return KERNEL_VERSION(major, minor, patch); + + return 0; } __u32 get_kernel_version(void)
diff --git a/tools/lib/bpf/libbpf_version.h b/tools/lib/bpf/libbpf_version.h index c446c0c..57b74ef 100644 --- a/tools/lib/bpf/libbpf_version.h +++ b/tools/lib/bpf/libbpf_version.h
@@ -4,6 +4,6 @@ #define __LIBBPF_VERSION_H #define LIBBPF_MAJOR_VERSION 1 -#define LIBBPF_MINOR_VERSION 8 +#define LIBBPF_MINOR_VERSION 9 #endif /* __LIBBPF_VERSION_H */
diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 78f92c3..f3f71c4 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c
@@ -2159,8 +2159,13 @@ static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, dst_sym->st_name = name_off; dst_sym->st_info = sym->st_info; dst_sym->st_other = sym->st_other; - dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; - dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; + if (sym_is_extern) { + dst_sym->st_shndx = SHN_UNDEF; + dst_sym->st_value = 0; + } else { + dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; + dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; + } dst_sym->st_size = sym->st_size; obj->sym_map[src_sym_idx] = dst_sym_idx; @@ -2274,6 +2279,24 @@ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *ob insn->imm += sec->dst_off / sizeof(struct bpf_insn); else insn->imm += sec->dst_off; + } else if (sym_type == R_BPF_64_ABS64 && + (sec->shdr->sh_flags & SHF_EXECINSTR)) { + /* + * A pointer to a static function in a data section, + * which is stored in place as an offset of the + * function in its section. Data sections are kept + * in the byte order of the object. + */ + void *ptr = dst_linked_sec->raw_data + dst_rel->r_offset; + __u64 off; + + memcpy(&off, ptr, sizeof(off)); + if (linker->swapped_endian) + off = bswap_64(off); + off += sec->dst_off; + if (linker->swapped_endian) + off = bswap_64(off); + memcpy(ptr, &off, sizeof(off)); } else { pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); return -EINVAL; @@ -2574,6 +2597,10 @@ static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) continue; dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; + if (!glob_sym->is_extern) { + Elf64_Sym *sym = get_sym_by_idx(linker, glob_sym->sym_idx); + dst_var->offset = sym->st_value; + } /* Because underlying BTF type might have * changed, so might its size have changed, so * re-calculate and update it in sec_var.
diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 1f3f332..9e16b8e 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h
@@ -11,6 +11,7 @@ #include <linux/bpf.h> #else #include <unistd.h> +#include <sys/param.h> #include <sys/syscall.h> #include <sys/mman.h> #include <linux/keyctl.h> @@ -127,7 +128,7 @@ static inline void skel_free(const void *p) * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value * depending on bpf_loader_ctx->flags. */ -static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) +static inline void skel_free_map_data(void *p, __u64 addr, size_t val_sz, __u32 max_entries) { if (addr != ~0ULL) kvfree(p); @@ -138,18 +139,20 @@ static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) */ } -static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) +static inline void *skel_prep_map_data(const void *val, size_t val_sz, __u32 max_entries, + size_t data_sz) { void *addr; - addr = kvmalloc(val_sz, GFP_KERNEL); + addr = kvmalloc(data_sz, GFP_KERNEL); if (!addr) return NULL; - memcpy(addr, val, val_sz); + memcpy(addr, val, data_sz); return addr; } -static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) +static inline void *skel_finalize_map_data(__u64 *init_val, size_t val_sz, __u32 max_entries, + int flags, int fd) { struct bpf_map *map; void *addr = NULL; @@ -172,9 +175,10 @@ static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int return addr; } -static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t val_sz, __u32 max_entries) { - (void)sz; + (void)val_sz; + (void)max_entries; kvfree(p); *init_val = ~0ULL; @@ -193,25 +197,39 @@ static inline void skel_free(void *p) free(p); } -static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) +static inline size_t skel_map_mmap_sz(size_t val_sz, __u32 max_entries) { - munmap(p, sz); + const long page_sz = sysconf(_SC_PAGE_SIZE); + size_t mmap_sz; + + mmap_sz = roundup(val_sz, 8) * max_entries; + mmap_sz = roundup(mmap_sz, page_sz); + return mmap_sz; } -static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) +static inline void skel_free_map_data(void *p, __u64 addr, size_t val_sz, __u32 max_entries) { + munmap(p, skel_map_mmap_sz(val_sz, max_entries)); +} + +static inline void *skel_prep_map_data(const void *val, size_t val_sz, __u32 max_entries, + size_t data_sz) +{ + size_t mmap_sz = skel_map_mmap_sz(val_sz, max_entries); void *addr; addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (addr == (void *) -1) return NULL; - memcpy(addr, val, val_sz); + memcpy(addr, val, data_sz); return addr; } -static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) +static inline void *skel_finalize_map_data(__u64 *init_val, size_t val_sz, __u32 max_entries, + int flags, int fd) { + size_t mmap_sz = skel_map_mmap_sz(val_sz, max_entries); void *addr; addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); @@ -220,11 +238,13 @@ static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int return addr; } -static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t val_sz, __u32 max_entries) { + size_t mmap_sz = skel_map_mmap_sz(val_sz, max_entries); + (void)init_val; - if (mprotect(p, sz, PROT_READ)) + if (mprotect(p, mmap_sz, PROT_READ)) return -errno; return 0; }
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 2e56e3a..ee9d1b6 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c
@@ -608,8 +608,12 @@ static bool has_nop_combo(int fd, long off) unsigned char nop_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; + long page_sz = getpagesize(); unsigned char buf[11]; + /* the kernel can't attach to a nop10 that crosses a page boundary */ + if ((off + 1) % page_sz + 10 > page_sz) + return false; if (pread(fd, buf, 11, off) != 11) return false; return memcmp(buf, nop_combo, 11) == 0;
diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 986a6389..b815bf0 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore
@@ -17,7 +17,6 @@ feature urandom_read test_sockmap -test_lirc_mode2_user flow_dissector_load test_tcpnotify_user test_libbpf
diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index ca1beae..75a9690 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64
@@ -1,2 +1,3 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions +probe_user # kprobe hit in trap
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 93c7071..1e4af62 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile
@@ -3,93 +3,18 @@ include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include -OBJCOPY ?= $(CROSS_COMPILE)objcopy - -CURDIR := $(abspath .) -TOOLSDIR := $(abspath ../../..) -LIBDIR := $(TOOLSDIR)/lib -BPFDIR := $(LIBDIR)/bpf -TOOLSINCDIR := $(TOOLSDIR)/include -TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include -BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool -APIDIR := $(TOOLSINCDIR)/uapi -ifneq ($(O),) -GENDIR := $(O)/include/generated -else -GENDIR := $(abspath ../../../../include/generated) -endif -GENHDR := $(GENDIR)/autoconf.h -PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config - -ifneq ($(wildcard $(GENHDR)),) - GENFLAGS := -DHAVE_GENHDR -endif - BPF_GCC ?= $(shell command -v bpf-gcc;) -ifdef ASAN -SAN_CFLAGS ?= -fsanitize=address -fno-omit-frame-pointer -else -SAN_CFLAGS ?= -endif -SAN_LDFLAGS ?= $(SAN_CFLAGS) -RELEASE ?= -OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0) - -LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null) -LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) SKIP_DOCS ?= SKIP_LLVM ?= SKIP_LIBBFD ?= SKIP_CRYPTO ?= -# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or -# benchmark compilation failure is fatal. Set to 0 to tolerate failures -# and continue building the remaining tests. -BPF_STRICT_BUILD ?= 1 -PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) - -ifeq ($(srctree),) -srctree := $(patsubst %/,%,$(dir $(CURDIR))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -endif - -COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ - -Wall -Werror -fno-omit-frame-pointer \ - -Wno-unused-but-set-variable \ - $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ - -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ - -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ - -I$(CURDIR)/libarena/include -LDFLAGS += $(SAN_LDFLAGS) -LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread - -PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") -PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) -LDLIBS += $(PCAP_LIBS) -CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) - -# Some utility functions use LLVM libraries -jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS) - -ifneq ($(LLVM),) -# Silence some warnings when compiled with clang -CFLAGS += -Wno-unused-command-line-argument -endif - # Check whether bpf cpu=v4 is supported or not by clang ifneq ($(shell $(CLANG) --target=bpf -mcpu=help 2>&1 | grep 'v4'),) CLANG_CPUV4 := 1 endif -# Check whether clang supports BPF address sanitizer (requires LLVM 22+) -CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ - $(CLANG) --target=bpf -fsanitize=kernel-address \ - -mllvm -asan-shadow-addr-space=1 \ - -x c -c - -o /dev/null 2>/dev/null && echo 1) - # Order correspond to 'make run_tests' order TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_progs \ test_sockmap \ @@ -101,16 +26,6 @@ ifneq ($(BPF_GCC),) TEST_GEN_PROGS += test_progs-bpf_gcc TEST_INST_SUBDIRS += bpf_gcc - -# The following tests contain C code that, although technically legal, -# triggers GCC warnings that cannot be disabled: declaration of -# anonymous struct types in function parameter lists. -progs/btf_dump_test_case_bitfields.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_namespacing.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_packing.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error - endif ifneq ($(CLANG_CPUV4),) @@ -118,12 +33,12 @@ TEST_INST_SUBDIRS += cpuv4 endif -TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) +TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected # Order correspond to 'make run_tests' order -TEST_PROGS := test_kmod.sh \ - test_lirc_mode2.sh \ - test_bpftool_build.sh \ +TEST_PROGS := test_bpftool_build.sh \ test_doc_build.sh \ test_xsk.sh \ test_xdp_features.sh @@ -140,7 +55,6 @@ bench \ flow_dissector_load \ test_cpp \ - test_lirc_mode2_user \ veristat \ xdp_features \ xdp_hw_metadata \ @@ -167,7 +81,17 @@ $(Q)$(MAKE) docs-clean endef +# Append build flags before lib.mk's user flags. References to +# Makefile.buildvars definitions expand when the flags are used. +# The arena-ASAN define is appended after lib.mk. +CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) $(CLANG_WARN_CFLAGS) +LDFLAGS += $(SAN_LDFLAGS) include ../lib.mk +include Makefile.buildvars + +BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool +HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a +BPF_TARGET_ENDIAN:=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) NON_CHECK_FEAT_TARGETS := clean docs-clean emit_tests CHECK_FEAT := $(filter-out $(NON_CHECK_FEAT_TARGETS),$(or $(MAKECMDGOALS), "none")) @@ -212,21 +136,9 @@ endif endif -SCRATCH_DIR := $(OUTPUT)/tools -BUILD_DIR := $(SCRATCH_DIR)/build -INCLUDE_DIR := $(SCRATCH_DIR)/include -BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a -ifneq ($(CROSS_COMPILE),) -HOST_BUILD_DIR := $(BUILD_DIR)/host -HOST_SCRATCH_DIR := $(OUTPUT)/host-tools -HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include -else -HOST_BUILD_DIR := $(BUILD_DIR) -HOST_SCRATCH_DIR := $(SCRATCH_DIR) -HOST_INCLUDE_DIR := $(INCLUDE_DIR) -endif -HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a -RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids +# Some utility functions use LLVM libraries +$(OUTPUT)/jit_disasm_helpers.o: CFLAGS += $(LLVM_CFLAGS) + VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ ../../../../vmlinux \ @@ -245,7 +157,7 @@ $(TEST_GEN_PROGS_EXTENDED)): %: $(OUTPUT)/% ; # sort removes libbpf duplicates when not cross-building -MAKE_DIRS := $(sort $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ +MAKE_DIRS := $(sort $(OUTPUT) $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ $(BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/bpftool \ $(HOST_BUILD_DIR)/resolve_btfids \ $(INCLUDE_DIR)) @@ -257,9 +169,11 @@ $(call msg,CC,,$@) $(Q)$(CC) $(CFLAGS) -c $(filter %.c,$^) $(LDLIBS) -o $@ +# Keep header prerequisites off the link command: clang treats +# bare headers as precompiled-header jobs and refuses -o. $(OUTPUT)/%:%.c $(call msg,BINARY,,$@) - $(Q)$(LINK.c) $^ $(LDLIBS) -o $@ + $(Q)$(LINK.c) $(filter-out %.h,$^) $(LDLIBS) -o $@ # LLVM's ld.lld doesn't support all the architectures, so use it only on x86 ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 riscv)) @@ -312,17 +226,6 @@ $(call msg,MOD,,$@) $(Q)$(if $(PERMISSIVE),if [ -f test_kmods/$(@F) ]; then )cp test_kmods/$(@F) $@$(if $(PERMISSIVE),; fi) - -DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -ifneq ($(CROSS_COMPILE),) -CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool -TRUNNER_BPFTOOL := $(CROSS_BPFTOOL) -USE_BOOTSTRAP := "" -else -TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) -USE_BOOTSTRAP := "bootstrap/" -endif - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -337,13 +240,29 @@ $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS) $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) -$(OUTPUT)/test_lirc_mode2_user: $(TESTING_HELPERS) -$(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) -$(OUTPUT)/test_maps: $(TESTING_HELPERS) +$(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) flow_dissector_load.h $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) -$(OUTPUT)/xsk.o: $(BPFOBJ) -BPFTOOL ?= $(DEFAULT_BPFTOOL) +# Defined ahead of the first rule that lists it as a prerequisite. +VERIFY_SIG_HDR := verification_cert.h + +# The userspace objects the runner instances link (RUNNER_OBJS, see +# Makefile.buildvars) are built here, once, so that no runner sub-make +# ever writes a file this Makefile also builds. +# +# Helper C files include headers installed to $(INCLUDE_DIR) by the +# $(TRUNNER_BPFTOOL) and $(BPFOBJ) targets. +RUNNER_OBJS := $(sort $(RUNNER_OBJS-test_progs) $(RUNNER_OBJS-test_maps)) +$(RUNNER_OBJS): $(VERIFY_SIG_HDR) $(BPFOBJ) $(TRUNNER_BPFTOOL) +$(OUTPUT)/test_progs.o: prog_tests/tests.h +$(OUTPUT)/test_maps.o: map_tests/tests.h + +# find_bit.c is the only runner source outside this directory, so it +# needs a rule of its own. +$(OUTPUT)/find_bit.o: $(TOOLSDIR)/lib/find_bit.c + $(call msg,CC,,$@) + $(Q)$(CC) $(CFLAGS) -c $< $(LDLIBS) -o $@ + $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \ @@ -411,7 +330,7 @@ # vmlinux.h is first dumped to a temporary file and then compared to # the previous version. This helps to avoid unnecessary re-builds of -# $(TRUNNER_BPF_OBJS) +# BPF objects. $(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) ifeq ($(VMLINUX_H),) $(call msg,GEN,,$@) @@ -423,7 +342,9 @@ endif $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ - $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \ $(TOOLSDIR)/lib/rbtree.c \ $(TOOLSDIR)/lib/zalloc.c \ $(TOOLSDIR)/lib/string.c \ @@ -450,360 +371,17 @@ $(shell $(1) $(2) -dM -E - </dev/null | grep -E 'MIPS(EL|EB)|_MIPS_SZ(PTR|LONG) |_MIPS_SIM |_ABI(O32|N32|64) ' | awk '{printf("-D%s=%s ", $$2, $$3)}') endef -# Determine target endianness. -IS_LITTLE_ENDIAN := $(shell $(CC) -dM -E - </dev/null | \ - grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') -MENDIAN:=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) -BPF_TARGET_ENDIAN:=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) - ifneq ($(CROSS_COMPILE),) CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%)) endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) -BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ - -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ - -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ - -I$(abspath $(OUTPUT)/../usr/include) \ - -std=gnu11 \ - -fno-strict-aliasing \ - -Wno-microsoft-anon-tag \ - -fms-extensions \ - -Wno-compare-distinct-pointer-types \ - -Wno-initializer-overrides \ - # -# TODO: enable me -Wsign-compare - CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) -$(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline -$(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline - -$(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h -$(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h - -# Build BPF object using Clang -# $1 - input .c file -# $2 - output .o file -# $3 - CFLAGS -# $4 - binary name -define CLANG_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef -# Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 -define CLANG_NOALU32_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef -# Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4 -define CLANG_CPUV4_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef -# Build BPF object using GCC -define GCC_BPF_BUILD_RULE - $(call msg,GCC-BPF,$4,$2) - $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef - -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c - -LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ - linked_vars.skel.h linked_maps.skel.h \ - test_subskeleton.skel.h test_subskeleton_lib.skel.h \ - test_usdt.skel.h tracing_multi.skel.h \ - tracing_multi_module.skel.h \ - tracing_multi_intersect.skel.h \ - tracing_multi_session.skel.h - -LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ - core_kern.c core_kern_overflow.c test_ringbuf.c \ - test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ - test_ringbuf_overwrite.c - -LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c - -# Generate both light skeleton and libbpf skeleton for these -LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c test_global_percpu_data.c -SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) - -test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o -linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o -linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o -linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o -# In the subskeleton case, we want the test_subskeleton_lib.subskel.h file -# but that's created as a side-effect of the skel.h generation. -test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o -test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o -test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o -xsk_xdp_progs.skel.h-deps := xsk_xdp_progs.bpf.o -xdp_hw_metadata.skel.h-deps := xdp_hw_metadata.bpf.o -xdp_features.skel.h-deps := xdp_features.bpf.o -tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o -tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o -tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o -tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o - -LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) -LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) - -HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ - $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ - $(addprefix $(BPFDIR)/, bpf_core_read.h \ - bpf_endian.h \ - bpf_helpers.h \ - bpf_tracing.h) - -# Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on -# $eval()) and pass control to DEFINE_TEST_RUNNER_RULES. -# Parameters: -# $1 - test runner base binary name (e.g., test_progs) -# $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) -define DEFINE_TEST_RUNNER - -LSKEL_SIGN := -S -k $(PRIVATE_KEY) -i $(VERIFICATION_CERT) -TRUNNER_OUTPUT := $(OUTPUT)$(if $2,/)$2 -TRUNNER_BINARY := $1$(if $2,-)$2 -TRUNNER_TEST_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.test.o, \ - $$(notdir $$(wildcard $(TRUNNER_TESTS_DIR)/*.c))) -TRUNNER_EXTRA_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ - $$(filter %.c,$(TRUNNER_EXTRA_SOURCES))) -TRUNNER_LIB_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ - $$(filter %.c,$(TRUNNER_LIB_SOURCES))) -TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES)) -TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h -TRUNNER_BPF_SRCS := $$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c)) -TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.bpf.o, $$(TRUNNER_BPF_SRCS)) -TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h, \ - $$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\ - $$(TRUNNER_BPF_SRCS))) -TRUNNER_BPF_LSKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS) $$(LSKELS_EXTRA)) -TRUNNER_BPF_SKELS_LINKED := $$(addprefix $$(TRUNNER_OUTPUT)/,$(LINKED_SKELS)) -TRUNNER_BPF_LSKELS_SIGNED := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS_SIGNED)) -TEST_GEN_FILES += $$(TRUNNER_BPF_OBJS) - -# Evaluate rules now with extra TRUNNER_XXX variables above already defined -$$(eval $$(call DEFINE_TEST_RUNNER_RULES,$1,$2)) - -endef - -# Using TRUNNER_XXX variables, provided by callers of DEFINE_TEST_RUNNER and -# set up by DEFINE_TEST_RUNNER itself, create test runner build rules with: -# $1 - test runner base binary name (e.g., test_progs) -# $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) -define DEFINE_TEST_RUNNER_RULES - -# Permissive build behaviour (skip-on-failure compile, partial-link) only -# applies to test_progs and its flavors; runners that use strong cross-object -# references (e.g. test_maps) keep strict semantics even when permissive. -# The check is inlined per-runner so $1 is substituted at $(call) time and -# the result is baked into each rule's recipe. - -ifeq ($($(TRUNNER_OUTPUT)-dir),) -$(TRUNNER_OUTPUT)-dir := y -$(TRUNNER_OUTPUT): - $$(call msg,MKDIR,,$$@) - $(Q)mkdir -p $$@ -endif - -# ensure we set up BPF objects generation rule just once for a given -# input/output directory combination -ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),) -$(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y -$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o: \ - $(TRUNNER_BPF_PROGS_DIR)/%.c \ - $(TRUNNER_BPF_PROGS_DIR)/*.h \ - $$(INCLUDE_DIR)/vmlinux.h \ - $(HEADERS_FOR_BPF_OBJS) \ - | $(TRUNNER_OUTPUT) $$(BPFOBJ) - $$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@, \ - $(TRUNNER_BPF_CFLAGS) \ - $$($$<-CFLAGS) \ - $$($$<-$2-CFLAGS),$(TRUNNER_BINARY)) - -$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) && \ - diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) && \ - $$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(<:.o=.linked1.o) $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) - -$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ - diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - -$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY) (signed)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ - diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - -$(LINKED_BPF_OBJS): %: $(TRUNNER_OUTPUT)/% - -# .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites -.SECONDEXPANSION: -$(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),for f in $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)); do \ - if [ ! -f $$$$f ]; then \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi; \ - done;) \ - printf ' %-12s %s\n' 'LINK-BPF' '[$(TRUNNER_BINARY)] $$(notdir $$(@:.skel.h=.bpf.o))' 1>&2; \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) && \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) && \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) && \ - diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) && \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2 && \ - $$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(@:.skel.h=.linked1.o) $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) - -# When the compiler generates a %.d file, only skel basenames (not -# full paths) are specified as prerequisites for corresponding %.o -# file. vpath directives below instruct make to search for skel files -# in TRUNNER_OUTPUT, if they are not present in the working directory. -vpath %.skel.h $(TRUNNER_OUTPUT) -vpath %.lskel.h $(TRUNNER_OUTPUT) -vpath %.subskel.h $(TRUNNER_OUTPUT) - -endif - -# ensure we set up tests.h header generation rule just once -ifeq ($($(TRUNNER_TESTS_DIR)-tests-hdr),) -$(TRUNNER_TESTS_DIR)-tests-hdr := y -$(TRUNNER_TESTS_HDR): $(TRUNNER_TESTS_DIR)/*.c - $$(call msg,TEST-HDR,$(TRUNNER_BINARY),$$@) - $$(shell (echo '/* Generated header, do not edit */'; \ - sed -n -E 's/^void (serial_)?test_([a-zA-Z0-9_]+)\((void)?\).*/DEFINE_TEST(\2)/p' \ - $(TRUNNER_TESTS_DIR)/*.c | sort ; \ - ) > $$@) -endif - -$(TRUNNER_OUTPUT)/resolve_btfids.test.o: $(RESOLVE_BTFIDS) $(TRUNNER_OUTPUT)/btf_data.bpf.o -$(TRUNNER_OUTPUT)/resolve_btfids.test.o: private TEST_NEEDS_BTFIDS = 1 - -# compile individual test files -# Note: we cd into output directory to ensure embedded BPF object is found -$(TRUNNER_TEST_OBJS): $(TRUNNER_OUTPUT)/%.test.o: \ - $(TRUNNER_TESTS_DIR)/%.c \ - | $(TRUNNER_OUTPUT)/%.test.d - $$(call msg,TEST-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)(cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),|| \ - ($(RM) $$@; printf ' %-12s %s\n' 'SKIP-TEST' '$$(notdir $$@)' 1>&2))) - $$(if $$(TEST_NEEDS_BTFIDS), \ - $(Q)if [ -f $$@ ]; then \ - $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(TRUNNER_BINARY)]" "$$(notdir $$@)"); \ - $(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@; \ - $(RESOLVE_BTFIDS) --patch_btfids $$@.BTF_ids $$@; \ - fi) - -$(TRUNNER_TEST_OBJS:.o=.d): $(TRUNNER_OUTPUT)/%.test.d: \ - $(TRUNNER_TESTS_DIR)/%.c \ - $(TRUNNER_EXTRA_HDRS) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) \ - $(TRUNNER_BPF_SKELS) \ - $(TRUNNER_BPF_LSKELS) \ - $(TRUNNER_BPF_LSKELS_SIGNED) \ - $(TRUNNER_BPF_SKELS_LINKED) - -ifeq ($(filter clean docs-clean emit_tests,$(MAKECMDGOALS)),) -include $(wildcard $(TRUNNER_TEST_OBJS:.o=.d)) -endif - -# add per extra obj CFGLAGS definitions -$(foreach N,$(patsubst $(TRUNNER_OUTPUT)/%.o,%,$(TRUNNER_EXTRA_OBJS)), \ - $(eval $(TRUNNER_OUTPUT)/$(N).o: CFLAGS += $($(N).c-CFLAGS))) - -$(TRUNNER_EXTRA_OBJS): $(TRUNNER_OUTPUT)/%.o: \ - %.c \ - $(TRUNNER_EXTRA_HDRS) \ - $(VERIFY_SIG_HDR) \ - $(TRUNNER_TESTS_HDR) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) - $$(call msg,EXT-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ - -$(TRUNNER_LIB_OBJS): $(TRUNNER_OUTPUT)/%.o:$(TOOLSDIR)/lib/%.c - $$(call msg,LIB-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ - -# non-flavored in-srctree builds receive special treatment, in particular, we -# do not need to copy extra resources (see e.g. test_btf_dump_case()) -$(TRUNNER_BINARY)-extras: $(TRUNNER_EXTRA_FILES) | $(TRUNNER_OUTPUT) -ifneq ($2:$(OUTPUT),:$(shell pwd)) - $$(call msg,EXT-COPY,$(TRUNNER_BINARY),$(TRUNNER_EXTRA_FILES)) - $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) $$^ $(TRUNNER_OUTPUT)/ -endif - -# some X.test.o files have runtime dependencies on Y.bpf.o files -$(OUTPUT)/$(TRUNNER_BINARY): | $(TRUNNER_BPF_OBJS) - -$(OUTPUT)/$(TRUNNER_BINARY): $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(wildcard $(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)) \ - $(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \ - $(TRUNNER_LIB_OBJS) \ - $(TRUNNER_BPFTOOL) \ - $(OUTPUT)/veristat \ - | $(TRUNNER_BINARY)-extras \ - $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$(TRUNNER_TEST_OBJS))) - $$(call msg,BINARY,,$$@) - $(Q)$$(CC) $$(CFLAGS) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(filter %.a %.o,$$(wildcard $(TRUNNER_TEST_OBJS)) $$(filter-out $(TRUNNER_TEST_OBJS),$$^)),$$(filter %.a %.o,$$^)),$$(filter %.a %.o,$$^)) $$(LDLIBS) $$(LLVM_LDLIBS) $$(LDFLAGS) $$(LLVM_LDFLAGS) -o $$@ - $(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool \ - $(OUTPUT)/$(if $2,$2/)bpftool - -endef - -VERIFY_SIG_SETUP := $(CURDIR)/verify_sig_setup.sh -VERIFY_SIG_HDR := verification_cert.h -VERIFICATION_CERT := $(BUILD_DIR)/signing_key.der -PRIVATE_KEY := $(BUILD_DIR)/signing_key.pem - -$(VERIFICATION_CERT) $(PRIVATE_KEY): $(VERIFY_SIG_SETUP) +# genkey writes both outputs. Pattern rules group them even on make +# versions before 4.3, preventing concurrent genkey invocations. +# Restrict the pattern to *_key files because genkey uses fixed names. +$(BUILD_DIR)/%_key.pem $(BUILD_DIR)/%_key.der: $(VERIFY_SIG_SETUP) $(Q)mkdir -p $(BUILD_DIR) $(Q)$(VERIFY_SIG_SETUP) genkey $(BUILD_DIR) @@ -825,108 +403,179 @@ BPF_TARGET_ENDIAN="$(BPF_TARGET_ENDIAN)" \ Q="$(Q)" -LIBARENA_BPF_DEPS := $(wildcard libarena/Makefile \ - libarena/include/* \ - libarena/include/libarena/* \ - libarena/src/* \ - libarena/selftests/* \ - libarena/*.bpf.o) +LIBARENA_COMMON_DEPS := $(wildcard libarena/Makefile \ + libarena/include/* \ + libarena/include/libarena/* \ + libarena/src/*) +LIBARENA_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/selftests/*) +LIBARENA_BENCH_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/benchs/*) -LIBARENA_SKEL := libarena/libarena.skel.h +LIBARENA_BENCH_SKEL := libarena/libarena_bench.skel.h $(LIBARENA_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena.skel.h $(LIBARENA_MAKE_ARGS) -ifneq ($(CLANG_HAS_ARENA_ASAN),) -LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h -CFLAGS += -DHAS_BPF_ARENA_ASAN +$(LIBARENA_BENCH_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BENCH_BPF_DEPS) | $(LIBARENA_SKEL) + +$(MAKE) -C libarena benchmarks $(LIBARENA_MAKE_ARGS) +ifneq ($(CLANG_HAS_ARENA_ASAN),) $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif -# Define test_progs test runner. -TRUNNER_TESTS_DIR := prog_tests -TRUNNER_BPF_PROGS_DIR := progs -TRUNNER_EXTRA_SOURCES := test_progs.c \ - cgroup_helpers.c \ - trace_helpers.c \ - network_helpers.c \ - testing_helpers.c \ - btf_helpers.c \ - cap_helpers.c \ - unpriv_helpers.c \ - sysctl_helpers.c \ - netlink_helpers.c \ - jit_disasm_helpers.c \ - io_helpers.c \ - test_loader.c \ - xsk.c \ - disasm.c \ - disasm_helpers.c \ - json_writer.c \ - $(VERIFY_SIG_HDR) \ - flow_dissector_load.h \ - ip_check_defrag_frags.h \ - bpftool_helpers.c \ - usdt_1.c usdt_2.c \ - $(LIBARENA_SKEL) \ - $(LIBARENA_ASAN_SKEL) -TRUNNER_LIB_SOURCES := find_bit.c -TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ - $(OUTPUT)/liburandom_read.so \ - $(OUTPUT)/xdp_synproxy \ - $(OUTPUT)/sign-file \ - $(OUTPUT)/uprobe_multi \ - $(TEST_KMOD_TARGETS) \ - ima_setup.sh \ - $(VERIFY_SIG_SETUP) \ - $(wildcard progs/btf_dump_test_case_*.c) \ - $(wildcard progs/*.bpf.o) -TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS -$(eval $(call DEFINE_TEST_RUNNER,test_progs)) +# #![no_std] looks for compiler_builtins too. Nothing of it is used. +ifneq ($(RUST_CORE),) +$(RUST_CORE): $(RUST_CORE_SRC) + $(call msg,RUSTC,,$@) + $(Q)mkdir -p $(@D) + +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< + +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ + $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - +endif -# Define test_progs-no_alu32 test runner. -TRUNNER_BPF_BUILD_RULE := CLANG_NOALU32_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -$(eval $(call DEFINE_TEST_RUNNER,test_progs,no_alu32)) +# Generated test list headers -# Define test_progs-cpuv4 test runner. +define gen_tests_hdr + $(call msg,TEST-HDR,,$@) + $(Q)(echo '/* Generated header, do not edit */'; \ + sed -n -E 's/^void (serial_)?test_([a-zA-Z0-9_]+)\((void)?\).*/DEFINE_TEST(\2)/p' \ + $(@D)/*.c | sort) > $@ +endef + +prog_tests/tests.h: $(wildcard prog_tests/*.c) + $(gen_tests_hdr) + +map_tests/tests.h: $(wildcard map_tests/*.c) + $(gen_tests_hdr) + +# Test runner instances, one sub-make each (see Makefile.runner). + +# Share LLVM probe results with runners. Pass CC explicitly to +# runners so it does not affect libbpf's compiler selection. +export LLVM_LDLIBS LLVM_LDFLAGS + +# Pass assembled flags on the command line so runner assignments +# cannot append them again. +RUNNER_MAKE := $(MAKE) --no-print-directory -f Makefile.runner \ + OUTPUT=$(OUTPUT) CC='$(CC)' CLANG='$(CLANG)' \ + CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' + +# Shared inputs prepared before runner sub-makes start. +# Also build veristat, which test_veristat executes. +RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ + $(TRUNNER_BPFTOOL) $(RESOLVE_BTFIDS) \ + $(OUTPUT)/veristat \ + $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ + $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ + prog_tests/tests.h map_tests/tests.h \ + $(RUNNER_OBJS) $(RUST_CORE) + +# Runtime fixtures for each test_progs flavor. +RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ + $(OUTPUT)/liburandom_read.so \ + $(OUTPUT)/xdp_synproxy \ + $(OUTPUT)/sign-file \ + $(OUTPUT)/uprobe_multi \ + $(TEST_KMOD_TARGETS) \ + ima_setup.sh \ + $(VERIFY_SIG_SETUP) \ + $(wildcard progs/btf_dump_test_case_*.c) \ + $(wildcard progs/*.bpf.o) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected + +# Prepare fixtures in parallel with runner compilation. +# Copy into flavor and out-of-tree directories; the in-tree +# default runner uses the originals. +RUNNER_EXTRA_DIRS := $(addprefix $(OUTPUT)/,$(TEST_INST_SUBDIRS)) +ifneq ($(realpath $(OUTPUT)),$(CURDIR)) +RUNNER_EXTRA_DIRS += $(OUTPUT) +test_progs: $(OUTPUT)/runner-extras +else +test_progs: $(RUNNER_EXTRA_FILES) +endif +RUNNER_EXTRAS := $(addsuffix /runner-extras,$(RUNNER_EXTRA_DIRS)) + +.PHONY: $(RUNNER_EXTRAS) +$(RUNNER_EXTRAS): $(RUNNER_EXTRA_FILES) + $(Q)mkdir -p $(@D) + $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) \ + $(RUNNER_EXTRA_FILES) $(@D)/ + +all: $(RUNNER_EXTRAS) +$(addprefix test_progs-,$(TEST_INST_SUBDIRS)): test_progs-%: \ + $(OUTPUT)/%/runner-extras + +# The main Makefile cannot tell whether $(OUTPUT)/test_progs is stale - +# only the runner sub-make knows its full dependency graph. FORCE makes +# the delegating rules below always run; their dependents still rebuild +# on mtime only. +FORCE: + +# Build default BPF outputs here for standalone consumers and install. +# Standalone consumers depend on individual skeletons; unflavored +# runners depend on the complete set before their sub-makes start. +RDIR := $(OUTPUT) +FLAVOR := +BINARY := test_progs +BPF_CC := $(CLANG) +BPF_CC_MSG := CLNG-BPF +BPF_SYS_INCLUDES := $(CLANG_SYS_INCLUDES) +BPF_CC_FLAGS := -O2 $(BPF_TARGET_ENDIAN) -mcpu=v3 +BPF_DEFINES := -DENABLE_ATOMICS_TESTS +include Makefile.skel + +DEFAULT_RUNNER_ARGS := RUNNER=test_progs FLAVOR= TESTS_DIR=prog_tests \ + BPF_CC='$(BPF_CC)' BPF_CC_MSG=$(BPF_CC_MSG) \ + BPF_SYS_INCLUDES='$(BPF_SYS_INCLUDES)' \ + BPF_CC_FLAGS='$(BPF_CC_FLAGS)' BPF_DEFINES=$(BPF_DEFINES) +CLANG_RUNNER_ARGS := BPF_CC='$(CLANG)' BPF_CC_MSG=CLNG-BPF \ + BPF_SYS_INCLUDES='$(CLANG_SYS_INCLUDES)' + +$(OUTPUT)/test_progs: $(RUNNER_PREREQS) $(BPF_OBJS) $(ALL_SKELS) FORCE + +$(Q)$(RUNNER_MAKE) $(DEFAULT_RUNNER_ARGS) + +$(OUTPUT)/test_progs-no_alu32: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=no_alu32 TESTS_DIR=prog_tests \ + $(CLANG_RUNNER_ARGS) BPF_CC_FLAGS='-O2 $(BPF_TARGET_ENDIAN) -mcpu=v2' + ifneq ($(CLANG_CPUV4),) -TRUNNER_BPF_BUILD_RULE := CLANG_CPUV4_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS -$(eval $(call DEFINE_TEST_RUNNER,test_progs,cpuv4)) +$(OUTPUT)/test_progs-cpuv4: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=cpuv4 TESTS_DIR=prog_tests \ + $(CLANG_RUNNER_ARGS) BPF_CC_FLAGS='-O2 $(BPF_TARGET_ENDIAN) -mcpu=v4' \ + BPF_DEFINES=-DENABLE_ATOMICS_TESTS endif -# Define test_progs BPF-GCC-flavored test runner. ifneq ($(BPF_GCC),) -TRUNNER_BPF_BUILD_RULE := GCC_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(call get_sys_includes,gcc,) -$(eval $(call DEFINE_TEST_RUNNER,test_progs,bpf_gcc)) +# Sys includes come from the *host* gcc, not $(BPF_GCC), as the rule +# this replaces had it. +GCC_SYS_INCLUDES := $(call get_sys_includes,gcc,) + +$(OUTPUT)/test_progs-bpf_gcc: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=bpf_gcc TESTS_DIR=prog_tests \ + BPF_CC='$(BPF_GCC)' BPF_CC_MSG=GCC-BPF \ + BPF_SYS_INCLUDES='$(GCC_SYS_INCLUDES)' \ + BPF_CC_FLAGS='-DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2' endif -# Define test_maps test runner. -TRUNNER_TESTS_DIR := map_tests -TRUNNER_BPF_PROGS_DIR := progs -TRUNNER_EXTRA_SOURCES := test_maps.c -TRUNNER_LIB_SOURCES := -TRUNNER_EXTRA_FILES := -TRUNNER_BPF_BUILD_RULE := $$(error no BPF objects should be built) -TRUNNER_BPF_CFLAGS := -$(eval $(call DEFINE_TEST_RUNNER,test_maps)) +# test_maps compiles map_tests/*.c against the default flavor's skeletons +$(OUTPUT)/test_maps: $(RUNNER_PREREQS) $(BPF_OBJS) \ + $(ALL_SKELS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_maps FLAVOR= TESTS_DIR=map_tests + +TEST_GEN_FILES += $(BPF_OBJS) # Define test_verifier test runner. # It is much simpler than test_maps/test_progs and sufficiently different from # them (e.g., test.h is using completely pattern), that it's worth just # explicitly defining all the rules explicitly. verifier/tests.h: verifier/*.c - $(shell ( cd verifier/; \ + $(Q)( cd verifier/; \ echo '/* Generated header, do not edit */'; \ echo '#ifdef FILL_ARRAY'; \ ls *.c 2> /dev/null | sed -e 's@\(.*\)@#include \"\1\"@'; \ echo '#endif' \ - ) > verifier/tests.h) + ) > verifier/tests.h $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ @@ -962,11 +611,11 @@ $(call msg,CXX,,$@) $(Q)$(CXX) $(CXXFLAGS) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@ -# Benchmark runner +# Benchmark runner. Every benchs/bench_*.c is compiled and linked in. +BENCH_OBJS := $(sort $(patsubst benchs/%.c,$(OUTPUT)/%.o,$(wildcard benchs/bench_*.c))) $(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ) $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ - ($(RM) $@; printf ' %-12s %s\n' 'SKIP-BENCH' '$(notdir $@)' 1>&2)) + $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(call skip_on_fail,BENCH) $(OUTPUT)/bench_rename.o: $(OUTPUT)/test_overhead.skel.h $(OUTPUT)/bench_trigger.o: $(OUTPUT)/trigger_bench.skel.h $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ @@ -986,6 +635,7 @@ $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_trie_map.skel.h $(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h $(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h +$(OUTPUT)/bench_libarena.o: $(LIBARENA_BENCH_SKEL) $(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm @@ -993,32 +643,11 @@ $(TESTING_HELPERS) \ $(TRACE_HELPERS) \ $(CGROUP_HELPERS) \ - $(OUTPUT)/bench_count.o \ - $(OUTPUT)/bench_rename.o \ - $(OUTPUT)/bench_trigger.o \ - $(OUTPUT)/bench_ringbufs.o \ - $(OUTPUT)/bench_bloom_filter_map.o \ - $(OUTPUT)/bench_bpf_loop.o \ - $(OUTPUT)/bench_bpf_for.o \ - $(OUTPUT)/bench_strncmp.o \ - $(OUTPUT)/bench_bpf_hashmap_full_update.o \ - $(OUTPUT)/bench_local_storage.o \ - $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o \ - $(OUTPUT)/bench_bpf_hashmap_lookup.o \ - $(OUTPUT)/bench_local_storage_create.o \ - $(OUTPUT)/bench_htab_mem.o \ - $(OUTPUT)/bench_bpf_crypto.o \ - $(OUTPUT)/bench_sockmap.o \ - $(OUTPUT)/bench_lpm_trie_map.o \ - $(OUTPUT)/bench_bpf_timing.o \ - $(OUTPUT)/bench_bpf_nop.o \ - $(OUTPUT)/bench_xdp_lb.o \ + $(BENCH_OBJS) \ $(OUTPUT)/usdt_1.o \ - $(OUTPUT)/usdt_2.o \ - # + $(OUTPUT)/usdt_2.o $(call msg,BINARY,,$@) - $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ - ($(RM) $@; printf ' %-12s %s\n' 'SKIP-LINK' '$(notdir $@) (some benchmarks may have been skipped)' 1>&2)) + $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(call skip_on_fail,LINK,,(some benchmarks may have been skipped)) # This works around GCC warning about snprintf truncating strings like: #
diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars new file mode 100644 index 0000000..ff3476b --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.buildvars
@@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Toolchain, path and flag definitions for the BPF selftests build, +# kept apart from the rules so that more than one makefile can include +# them. Included by the Makefile after ../lib.mk and by Makefile.runner +# in its place - the note above the Makefile's include explains how the +# flags are ordered around it. + +TOOLSDIR := $(abspath ../../..) +LIBDIR := $(TOOLSDIR)/lib +BPFDIR := $(LIBDIR)/bpf +TOOLSINCDIR := $(TOOLSDIR)/include +TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include +APIDIR := $(TOOLSINCDIR)/uapi +ifneq ($(O),) +GENDIR := $(O)/include/generated +else +GENDIR := $(abspath ../../../../include/generated) +endif +GENHDR := $(GENDIR)/autoconf.h +PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config + +ifneq ($(wildcard $(GENHDR)),) + GENFLAGS := -DHAVE_GENHDR +endif + +ifdef ASAN +SAN_CFLAGS ?= -fsanitize=address -fno-omit-frame-pointer +else +SAN_CFLAGS ?= +endif +SAN_LDFLAGS ?= $(SAN_CFLAGS) +RELEASE ?= +OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0) + +LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null) +LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) + +# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or +# benchmark compilation failure is fatal. Set to 0 to tolerate failures +# and continue building the remaining tests. +BPF_STRICT_BUILD ?= 1 +PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) + +# Permissive-mode recipe suffix: on failure, remove the target, emit a +# SKIP marker and report success so the rest of the build continues. +# $(1) - SKIP tag (BPF, TEST, BENCH, SKEL, LINK, ...) +# $(2) - optional extra files to remove alongside the target +# $(3) - optional note appended to the SKIP message +skip_on_fail = $(if $(PERMISSIVE),|| { $(RM) $@ $(2); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)$(if $(3), $(3))' 1>&2; }) + +# Permissive-mode recipe prefix: when a prerequisite object is missing +# (its compile failed and was skipped), remove the target and $(3), +# emit a SKIP marker and report success without running the rest of +# the recipe. +# $(1) - SKIP tag +# $(2) - input files to check +# $(3) - optional extra files to remove alongside the target +skip_if_missing = $(if $(PERMISSIVE),for f in $(2); do [ -f $$f ] || { \ + $(RM) $@ $(3); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)' 1>&2; exit 0; }; done;) + +ifeq ($(srctree),) +srctree := $(patsubst %/,%,$(dir $(CURDIR))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +endif + +COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ + -Wall -Werror -fno-omit-frame-pointer \ + -Wno-unused-but-set-variable \ + $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ + -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ + -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ + -I$(CURDIR)/libarena/include +LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread + +PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") +PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) +LDLIBS += $(PCAP_LIBS) + +ifneq ($(LLVM),) +# Silence some warnings when compiled with clang +CLANG_WARN_CFLAGS := -Wno-unused-command-line-argument +endif + +# Check whether clang supports BPF address sanitizer (requires LLVM 22+). +# This runs after ../lib.mk has resolved CLANG from LLVM=<suffix or path>, +# so it probes the compiler that builds the BPF objects. +CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ + $(CLANG) --target=bpf -fsanitize=kernel-address \ + -mllvm -asan-shadow-addr-space=1 \ + -x c -c - -o /dev/null 2>/dev/null && echo 1) + +SCRATCH_DIR := $(OUTPUT)/tools +BUILD_DIR := $(SCRATCH_DIR)/build +INCLUDE_DIR := $(SCRATCH_DIR)/include +BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a +ifneq ($(CROSS_COMPILE),) +HOST_BUILD_DIR := $(BUILD_DIR)/host +HOST_SCRATCH_DIR := $(OUTPUT)/host-tools +HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include +else +HOST_BUILD_DIR := $(BUILD_DIR) +HOST_SCRATCH_DIR := $(SCRATCH_DIR) +HOST_INCLUDE_DIR := $(INCLUDE_DIR) +endif +RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids + +# Programs in Rust are built by upstream rustc. It has no prebuilt core for +# the bpf target, so it has to come with the source of core: +# rustup component add rust-src +# core is built as edition 2024, which it is since rustc 1.87. +# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be +# 23 or newer and not older than LLVM of rustc. +# Otherwise RUST_CORE is empty and the tests are skipped. +RUSTC ?= rustc +RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ + /lib/rustlib/src/rust/library/core/src/lib.rs) +ifneq ($(RUST_CORE_SRC),) +ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ + llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ + [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ + [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ + 2>/dev/null && echo y),y) +RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib +endif +endif +# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. +# panic=abort is a stop gap until panic=unwind is supported. +RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ + --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ + -L $(dir $(RUST_CORE)) + +DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool +ifneq ($(CROSS_COMPILE),) +CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool +TRUNNER_BPFTOOL := $(CROSS_BPFTOOL) +USE_BOOTSTRAP := "" +else +TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) +USE_BOOTSTRAP := "bootstrap/" +endif + +BPFTOOL ?= $(DEFAULT_BPFTOOL) + +# Determine target endianness. +IS_LITTLE_ENDIAN := $(shell $(CC) -dM -E - </dev/null | \ + grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') +MENDIAN:=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) + +BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ + -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ + -I$(abspath $(OUTPUT)/../usr/include) \ + -std=gnu11 \ + -fno-strict-aliasing \ + -Wno-microsoft-anon-tag \ + -fms-extensions \ + -Wno-compare-distinct-pointer-types \ + -Wno-initializer-overrides \ + # +# TODO: enable me -Wsign-compare + +VERIFY_SIG_SETUP := $(CURDIR)/verify_sig_setup.sh +VERIFICATION_CERT := $(BUILD_DIR)/signing_key.der +PRIVATE_KEY := $(BUILD_DIR)/signing_key.pem + +LIBARENA_SKEL := libarena/libarena.skel.h + +ifneq ($(CLANG_HAS_ARENA_ASAN),) +LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h +CFLAGS += -DHAS_BPF_ARENA_ASAN +endif + +RUNNER_OBJS-test_progs := $(addprefix $(OUTPUT)/, \ + test_progs.o cgroup_helpers.o trace_helpers.o \ + network_helpers.o testing_helpers.o btf_helpers.o \ + cap_helpers.o unpriv_helpers.o sysctl_helpers.o \ + netlink_helpers.o jit_disasm_helpers.o io_helpers.o \ + test_loader.o xsk.o disasm.o disasm_helpers.o \ + json_writer.o bpftool_helpers.o usdt_1.o usdt_2.o \ + find_bit.o) +RUNNER_OBJS-test_maps := $(addprefix $(OUTPUT)/, \ + test_maps.o testing_helpers.o)
diff --git a/tools/testing/selftests/bpf/Makefile.runner b/tools/testing/selftests/bpf/Makefile.runner new file mode 100644 index 0000000..d266c42 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.runner
@@ -0,0 +1,125 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Build one test_progs flavor or test_maps. +# Invoked by the main Makefile after shared build inputs are ready. +# Flavored outputs live in $(OUTPUT)/$(FLAVOR); unflavored outputs +# share $(OUTPUT) with the main Makefile. +# +# Command-line parameters: +# OUTPUT selftests output directory, resolved by lib.mk +# RUNNER test_progs | test_maps +# FLAVOR empty | no_alu32 | cpuv4 | bpf_gcc +# TESTS_DIR prog_tests | map_tests +# BPF_* see Makefile.skel; BPF_CC is empty for test_maps +# CC, CLANG compilers resolved by the main Makefile +# CFLAGS, LDFLAGS assembled flags, including lib.mk's additions +# +# Exported by the main Makefile: +# LLVM_LDLIBS, LLVM_LDFLAGS: LLVM feature-probe results + +include ../../../build/Build.include +include ../../../scripts/Makefile.arch +include ../../../scripts/Makefile.include + +# Same message helpers as ../lib.mk, which only the top level includes. +ifeq ($(V),1) +Q = +msg = +else +Q = @ +msg = @printf ' %-8s%s %s%s\n' "$(1)" "$(if $(2), [$(2)])" "$(notdir $(3))" "$(if $(4), $(4))"; +MAKEFLAGS += --no-print-directory +endif + +include Makefile.buildvars + +BINARY := $(RUNNER)$(if $(FLAVOR),-$(FLAVOR)) +RDIR := $(OUTPUT)$(if $(FLAVOR),/$(FLAVOR)) + +all: $(OUTPUT)/$(BINARY) +.PHONY: all + +# Delete partially updated (corrupted) files on error +.DELETE_ON_ERROR: + +# No built-in rules: every file built here has an explicit rule, and the +# objects that come from the main Makefile have to be an error when +# missing rather than a silent rebuild by the built-in %.o recipes. +MAKEFLAGS += -r + +# Permissive build behaviour (skip-on-failure compile, partial-link) only +# applies to test_progs and its flavors; runners that use strong cross-object +# references (e.g. test_maps) keep strict semantics even when permissive. +PERMISSIVE_TESTS := $(if $(filter test_progs,$(RUNNER)),$(PERMISSIVE)) + +$(RDIR): + $(call msg,MKDIR,,$@) + $(Q)mkdir -p $@ + +# Per-runner sources + +ifeq ($(RUNNER),test_progs) +GENERATED_HDRS := $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) +endif + +TEST_SRCS := $(notdir $(wildcard $(TESTS_DIR)/*.c)) +TEST_OBJS := $(patsubst %.c,$(RDIR)/%.test.o,$(TEST_SRCS)) +TEST_DEPS := $(TEST_OBJS:.o=.d) + +# BPF objects and skeletons - rules shared with the main Makefile + +include Makefile.skel + +# When the compiler generates a %.d file, only skel basenames (not +# full paths) are specified as prerequisites for corresponding %.o +# file. vpath directives below instruct make to search for skel files +# in $(RDIR), if they are not present in the working directory. +vpath %.skel.h $(RDIR) +vpath %.lskel.h $(RDIR) +vpath %.subskel.h $(RDIR) + +# Test objects and runner binary + +$(RDIR)/resolve_btfids.test.o: $(RESOLVE_BTFIDS) $(RDIR)/btf_data.bpf.o +$(RDIR)/resolve_btfids.test.o: private TEST_NEEDS_BTFIDS = 1 + +# compile individual test files +# Note: we cd into output directory to ensure embedded BPF object is found +# The %.test.d dependency files are a side effect of the -MMD below; the +# separate no-recipe rule for them orders the first compilation after +# skeleton generation and re-orders test object compilation when headers +# change. +$(TEST_OBJS): $(RDIR)/%.test.o: $(TESTS_DIR)/%.c | $(RDIR)/%.test.d + $(call msg,TEST-OBJ,$(BINARY),$@) + $(Q)(cd $(@D) && $(CC) -I. $(CFLAGS) -MMD -MT $@ -c $(CURDIR)/$< $(LDLIBS) -o $(@F)) $(if \ + $(PERMISSIVE_TESTS),$(call skip_on_fail,TEST)) + $(if $(TEST_NEEDS_BTFIDS), \ + $(Q)if [ -f $@ ]; then \ + $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(BINARY)]" "$(notdir $@)"); \ + $(RESOLVE_BTFIDS) --btf $(RDIR)/btf_data.bpf.o $@; \ + $(RESOLVE_BTFIDS) --patch_btfids $@.BTF_ids $@; \ + fi) + +$(TEST_DEPS): $(RDIR)/%.test.d: $(TESTS_DIR)/%.c $(GENERATED_HDRS) $(BPFOBJ) \ + | $(RDIR) $(ALL_SKELS) + +include $(wildcard $(TEST_DEPS)) + +cmd_runner_link = $(CC) $(CFLAGS) \ + $(if $(PERMISSIVE_TESTS),$(wildcard $(TEST_OBJS)) \ + $(filter-out $(TEST_OBJS),$(filter %.a %.o,$^)), \ + $(filter %.a %.o,$^)) \ + $(LDLIBS) $(LLVM_LDLIBS) $(LDFLAGS) $(LLVM_LDFLAGS) -o $@ + +# BPF objects are also needed at runtime. +# In permissive mode, existing test objects are normal prerequisites +# so edits trigger relinking. Order-only prerequisites attempt all +# tests; the recipe links the objects that exist afterward. +$(OUTPUT)/$(BINARY): $(if $(PERMISSIVE_TESTS),$(wildcard $(TEST_OBJS)),$(TEST_OBJS)) \ + $(RUNNER_OBJS-$(RUNNER)) $(BPFOBJ) \ + $(TRUNNER_BPFTOOL) \ + | $(BPF_OBJS) \ + $(if $(PERMISSIVE_TESTS),$(TEST_OBJS)) + $(call msg,BINARY,,$@) + $(Q)$(cmd_runner_link) + $(Q)ln -sf $(if $(FLAVOR),..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool $(RDIR)/bpftool
diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel new file mode 100644 index 0000000..6570c48 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.skel
@@ -0,0 +1,167 @@ +# SPDX-License-Identifier: GPL-2.0 +# BPF object and skeleton rules for one flavor. +# Included by the main Makefile for default outputs and by runners +# for their flavor. Unflavored runners require the default outputs first. +# +# Parameters (variables defined by the includer): +# RDIR output directory of this flavor +# FLAVOR flavor name, empty for the default +# BINARY runner binary name, for build-log messages +# BPF_CC BPF compiler +# BPF_CC_MSG build-log tag (CLNG-BPF/GCC-BPF) +# BPF_CC_FLAGS per-flavor compiler flags +# BPF_SYS_INCLUDES system include flags for $(BPF_CC) +# BPF_DEFINES extra defines (-DENABLE_ATOMICS_TESTS) + +# BPF objects and skeletons + +ifneq ($(BPF_CC),) + +BPF_SRCS := $(notdir $(wildcard progs/*.c)) +BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) + +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ + data_in_arena_extern.c data_in_arena_nomap.c + +LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ + linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ + test_subskeleton.skel.h test_subskeleton_lib.skel.h \ + test_usdt.skel.h tracing_multi.skel.h \ + tracing_multi_module.skel.h \ + tracing_multi_intersect.skel.h \ + tracing_multi_session.skel.h + +LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ + core_kern.c core_kern_overflow.c test_ringbuf.c \ + test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ + test_ringbuf_overwrite.c callx_rodata.c + +LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c + +# Generate both light skeleton and libbpf skeleton for these +LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ + kfunc_call_test_subprog.c test_global_percpu_data.c +SKEL_BLACKLIST += $(LSKELS) $(LSKELS_SIGNED) + +test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o +linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o +linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o +linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o +linked_arena.skel.h-deps := linked_arena1.bpf.o linked_arena2.bpf.o +# In the subskeleton case, we want the test_subskeleton_lib.subskel.h file +# but that's created as a side-effect of the skel.h generation. +test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o +test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o +test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o +tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o +tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o + +LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) +LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) + +SKELS := $(patsubst %.c,$(RDIR)/%.skel.h, \ + $(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),$(BPF_SRCS))) +LSKELS_H := $(patsubst %.c,$(RDIR)/%.lskel.h,$(LSKELS) $(LSKELS_EXTRA)) +LSKELS_SIGNED_H := $(patsubst %.c,$(RDIR)/%.lskel.h,$(LSKELS_SIGNED)) +LINKED_SKELS_H := $(addprefix $(RDIR)/,$(LINKED_SKELS)) +ALL_SKELS := $(SKELS) $(LSKELS_H) $(LSKELS_SIGNED_H) $(LINKED_SKELS_H) + +HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ + $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ + $(addprefix $(BPFDIR)/, bpf_core_read.h \ + bpf_endian.h \ + bpf_helpers.h \ + bpf_tracing.h) + +# The following tests contain C code that, although technically legal, +# triggers GCC warnings that cannot be disabled: declaration of +# anonymous struct types in function parameter lists. +progs/btf_dump_test_case_bitfields.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_namespacing.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_packing.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error + +cmd_bpf_compile = $(BPF_CC) $(BPF_CFLAGS) $(BPF_SYS_INCLUDES) $(BPF_DEFINES) \ + $($<-CFLAGS) $($<-$(FLAVOR)-CFLAGS) $(BPF_CC_FLAGS) -c $< -o $@ $(call skip_on_fail,BPF) + +$(BPF_OBJS): $(RDIR)/%.bpf.o: progs/%.c progs/*.h \ + $(INCLUDE_DIR)/vmlinux.h $(HEADERS_FOR_BPF_OBJS) \ + | $(RDIR) $(BPFOBJ) + $(call msg,$(BPF_CC_MSG),$(BINARY),$@) + $(Q)$(cmd_bpf_compile) + +GEN_SKEL := ./gen_bpf_skel.sh + +cmd_bpf_gen_skel = $(call skip_if_missing,SKEL,$<,$(@:.skel.h=.subskel.h))printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $* \ + --skel $@ --subskel $(@:.skel.h=.subskel.h) $< $(call \ + skip_on_fail,SKEL,$(@:.skel.h=.subskel.h)) + +$(SKELS): $(RDIR)/%.skel.h: $(RDIR)/%.bpf.o $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_gen_skel) + +cmd_bpf_gen_lskel = $(call skip_if_missing,SKEL,$<)printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $*_lskel --lskel \ + --skel $@ $< $(call skip_on_fail,SKEL) + +$(LSKELS_H): $(RDIR)/%.lskel.h: $(RDIR)/%.bpf.o $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_gen_lskel) + +cmd_bpf_gen_signed_lskel = $(call skip_if_missing,SKEL,$<)printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY) (signed)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) PRIVATE_KEY=$(PRIVATE_KEY) \ + VERIFICATION_CERT=$(VERIFICATION_CERT) \ + $(GEN_SKEL) --sign --name $*_lskel \ + --skel $@ $< $(call skip_on_fail,SKEL) + +$(LSKELS_SIGNED_H): $(RDIR)/%.lskel.h: $(RDIR)/%.bpf.o $(BPFTOOL) \ + $(GEN_SKEL) $(PRIVATE_KEY) $(VERIFICATION_CERT) | $(RDIR) + $(Q)$(cmd_bpf_gen_signed_lskel) + +cmd_bpf_link_skel = $(call skip_if_missing,SKEL,$(addprefix $(RDIR)/,\ + $($*.skel.h-deps)),$(@:.skel.h=.subskel.h))printf \ + ' %-8s%s %s\n' 'LINK-BPF' ' [$(BINARY)]' '$(notdir $(@:.skel.h=.bpf.o))'; \ + printf ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $* \ + --skel $@ --subskel $(@:.skel.h=.subskel.h) \ + $(addprefix $(RDIR)/,$($*.skel.h-deps)) $(call \ + skip_on_fail,SKEL,$(@:.skel.h=.subskel.h)) + +# .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites +.SECONDEXPANSION: +$(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) \ + $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_link_skel) + +# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. +ifneq ($(RUST_CORE),) +ifeq ($(BPF_CC),$(CLANG)) +RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) +BPF_OBJS += $(RUST_OBJS) + +$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) + $(call msg,RUSTC,$(BINARY),$@) + +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ + $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ + -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ + $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) +endif +endif + +# Programs in C++. No skeletons: gcc doesn't build them. +ifeq ($(BPF_CC),$(CLANG)) +CXX_OBJS := $(patsubst progs/%.cpp,$(RDIR)/%.bpf.o,$(wildcard progs/*.cpp)) +BPF_OBJS += $(CXX_OBJS) + +$(CXX_OBJS): $(RDIR)/%.bpf.o: progs/%.cpp | $(RDIR) + $(call msg,$(BPF_CC_MSG),$(BINARY),$@) + $(Q)$(BPF_CC) -x c++ -g -Wall -Werror -fexceptions -funwind-tables -fno-rtti \ + $(BPF_CC_FLAGS) -c $< -o $@ $(call skip_on_fail,BPF) +endif + +endif # BPF_CC
diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 07c8344..75ff114 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst
@@ -187,7 +187,7 @@ ... 15: (bf) r2 = r7 16: (0f) r2 += r1 - ; BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol); + ; BPF_SEQ_PRINTF(seq, "%p %-3d ", s, s->sk_protocol); 17: (7b) *(u64 *)(r7 +0) = r2 only read is supported
diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 465233e..d5f4941 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c
@@ -288,6 +288,7 @@ extern struct argp bench_crypto_argp; extern struct argp bench_sockmap_argp; extern struct argp bench_lpm_trie_map_argp; extern struct argp bench_xdp_lb_argp; +extern struct argp bench_libarena_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, @@ -306,6 +307,7 @@ static const struct argp_child bench_parsers[] = { { &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 }, { &bench_lpm_trie_map_argp, 0, "LPM trie map benchmark", 0 }, { &bench_xdp_lb_argp, 0, "XDP load-balancer benchmark", 0 }, + { &bench_libarena_argp, 0, "libarena allocator benchmark", 0 }, {}, }; @@ -585,6 +587,8 @@ extern const struct bench bench_lpm_trie_delete; extern const struct bench bench_lpm_trie_free; extern const struct bench bench_bpf_nop; extern const struct bench bench_xdp_lb; +extern const struct bench bench_libarena_malloc; +extern const struct bench bench_libarena_calloc; static const struct bench *benchs[] = { &bench_count_global, @@ -669,6 +673,8 @@ static const struct bench *benchs[] = { &bench_lpm_trie_free, &bench_bpf_nop, &bench_xdp_lb, + &bench_libarena_malloc, + &bench_libarena_calloc, }; static void find_benchmark(void)
diff --git a/tools/testing/selftests/bpf/benchs/bench_libarena.c b/tools/testing/selftests/bpf/benchs/bench_libarena.c new file mode 100644 index 0000000..24e4322 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_libarena.c
@@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <argp.h> +#include <limits.h> +#include <string.h> + +#include "bench.h" + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> +#include <libarena/userspace.h> + +#include "libarena/libarena_bench.skel.h" + +static struct { + __u64 alloc_size; + __u64 nallocs; +} args = { + .alloc_size = 64, + .nallocs = 10000, +}; + +static struct { + struct libarena_bench *skel; + int bench_fd; + int reset_fd; +} ctx; + +enum { + ARG_LIBARENA_ALLOC_SIZE = 12000, + ARG_LIBARENA_NALLOCS, +}; + +static const struct argp_option opts[] = { + { "alloc_size", ARG_LIBARENA_ALLOC_SIZE, "BYTES", 0, + "Size of each arena allocation" }, + { "nallocs", ARG_LIBARENA_NALLOCS, "ITERS", 0, + "Number of allocation per measurement" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + unsigned long value; + + switch (key) { + case ARG_LIBARENA_ALLOC_SIZE: + value = strtoull(arg, NULL, 10); + if (!value || value >= UINT_MAX) { + fprintf(stderr, "invalid alloc_size: %ld", value); + argp_usage(state); + } + args.alloc_size = value; + break; + case ARG_LIBARENA_NALLOCS: + args.nallocs = strtoull(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_libarena_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumers\n"); + exit(1); + } + + if (env.producer_cnt != 1) { + fprintf(stderr, "benchmark supports exactly one producer\n"); + exit(1); + } +} + +static void setup_common(void) +{ + struct arena_alloc_reserve_args reserve_args = { + .nr_pages = ARENA_RESERVE_PAGES_DFL, + }; + int err; + + setup_libbpf(); + + ctx.skel = libarena_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open and load skeleton\n"); + exit(1); + } + + err = libarena_run_prog_args( + bpf_program__fd(ctx.skel->progs.arena_alloc_reserve), + &reserve_args, sizeof(reserve_args)); + if (err) { + fprintf(stderr, "failed to reserve arena pages: %d\n", err); + exit(1); + } + + err = libarena_run_prog( + bpf_program__fd(ctx.skel->progs.arena_buddy_reset)); + if (err) { + fprintf(stderr, "failed to initialize arena allocator: %d\n", err); + exit(1); + } + + ctx.skel->bss->bench_alloc_size = args.alloc_size; + ctx.skel->bss->bench_nallocs = args.nallocs; + ctx.reset_fd = bpf_program__fd(ctx.skel->progs.arena_buddy_reset); +} + +static void malloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_malloc); +} + +static void calloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_calloc); +} + +static void *producer(void *input) +{ + int err; + + while (true) { + err = libarena_run_prog(ctx.bench_fd); + if (err) { + fprintf(stderr, "libarena benchmark failed: %d\n", err); + exit(1); + } + + err = libarena_run_prog(ctx.reset_fd); + if (err) { + fprintf(stderr, "libarena alloc reset failed: %d\n", err); + exit(1); + } + } + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->duration_ns = atomic_swap(&ctx.skel->bss->bench_duration_ns, 0); + res->hits = atomic_swap(&ctx.skel->bss->bench_hits, 0); +} + +static void report_progress(int iter, struct bench_res *res, long delta_ns) +{ + double latency_ns = 0.0; + + if (res->hits) + latency_ns = res->duration_ns / (double)res->hits; + + printf("Iter %3d (%7.3lfus): latency %8.3lf ns/op (%ld allocations)\n", + iter, (delta_ns - 1000000000) / 1000.0, latency_ns, res->hits); +} + +static void report_final(struct bench_res res[], int res_cnt) +{ + unsigned long duration_ns = 0; + long hits = 0; + int i; + + for (i = 0; i < res_cnt; i++) { + duration_ns += res[i].duration_ns; + hits += res[i].hits; + } + + if (!hits || !res_cnt) { + printf("Summary: no runs measured\n"); + return; + } + + printf("Summary: %.3lf ns/op, %.0lf invocations for %u allocations/invocation)\n", + duration_ns / (double)hits, hits / (double)res_cnt, + ctx.skel->bss->bench_nallocs); +} + +const struct bench bench_libarena_malloc = { + .name = "libarena-malloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = malloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; + +const struct bench bench_libarena_calloc = { + .name = "libarena-calloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = calloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +};
diff --git a/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh new file mode 100755 index 0000000..10afe4d --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh
@@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +RUN_BENCH="./bench -d3 -q" + +summarize_libarena() +{ + local bench="$1" + local summary + + summary=$(printf '%s\n' "$2" | tail -n1) + summary=${summary#Summary: } + printf "%-20s %s\n" "$bench" "$summary" +} + +header "libarena sequential malloc\n" + +for size in 16 64 256 1024 4096; do +subtitle "allocation size: $size" +printf "\t-------------------\n" + for nallocs in 10 50 100 500 1000 5000 10000; do + summarize_libarena "malloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-malloc)" + summarize_libarena "calloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-calloc)" + done +done
diff --git a/tools/testing/selftests/bpf/bpf_kfuncs.h b/tools/testing/selftests/bpf/bpf_kfuncs.h index ae71e9b..1b408fe 100644 --- a/tools/testing/selftests/bpf/bpf_kfuncs.h +++ b/tools/testing/selftests/bpf/bpf_kfuncs.h
@@ -78,18 +78,29 @@ extern void bpf_key_put(struct bpf_key *key) __ksym; extern int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_ptr, const struct bpf_dynptr *sig_ptr, struct bpf_key *trusted_keyring) __ksym; - -struct dentry; -/* Description +/* + * Description * Returns xattr of a dentry * Returns * Error code */ +struct dentry; extern int bpf_get_dentry_xattr(struct dentry *dentry, const char *name, struct bpf_dynptr *value_ptr) __ksym __weak; - extern int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str, const struct bpf_dynptr *value_p, int flags) __ksym __weak; extern int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str) __ksym __weak; +/* + * Description + * Attach a xattr to an inode that is being created, from a program on the + * inode_init_security LSM hook. *xattrs* and *xattr_count* must be the + * hook's own arguments, passed through unmodified. + * Returns + * 0 on success, a negative value on error + */ +struct xattr; +extern int bpf_inode_init_xattr(struct xattr *xattrs, int *xattr_count, + const char *name__str, + const struct bpf_dynptr *value_p) __ksym __weak; #endif
diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected new file mode 100644 index 0000000..5470b2b --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected
@@ -0,0 +1,48 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +enum { + E0 = 1, +}; + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */
diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected new file mode 100644 index 0000000..4310adc --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected
@@ -0,0 +1,48 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +enum { + E0 = 1, +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */
diff --git a/tools/testing/selftests/bpf/bpftool_helpers.c b/tools/testing/selftests/bpf/bpftool_helpers.c index 0a2a4f0..9380b24 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.c +++ b/tools/testing/selftests/bpf/bpftool_helpers.c
@@ -11,7 +11,7 @@ #define BPFTOOL_DEFAULT_PATH "tools/sbin/bpftool" -static int detect_bpftool_path(char *buffer, size_t size) +int detect_bpftool_path(char *buffer, size_t size) { char tmp[PATH_MAX]; const char *env_path; @@ -53,6 +53,7 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) static char bpftool_path[PATH_MAX] = {}; bool suppress_output = !(output_buf && output_max_len); char command[BPFTOOL_FULL_CMD_MAX_LEN]; + size_t n; FILE *f; int ret; @@ -68,8 +69,10 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) if (!f) return 1; - if (!suppress_output) - fread(output_buf, 1, output_max_len, f); + if (!suppress_output) { + n = fread(output_buf, 1, output_max_len - 1, f); + output_buf[n] = '\0'; + } ret = pclose(f); return ret;
diff --git a/tools/testing/selftests/bpf/bpftool_helpers.h b/tools/testing/selftests/bpf/bpftool_helpers.h index dec1ba2..17e3be8 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.h +++ b/tools/testing/selftests/bpf/bpftool_helpers.h
@@ -7,5 +7,6 @@ #define MAX_BPFTOOL_CMD_LEN (256) +int detect_bpftool_path(char *buffer, size_t size); int run_bpftool_command(char *args); int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_len);
diff --git a/tools/testing/selftests/bpf/btf_helpers.c b/tools/testing/selftests/bpf/btf_helpers.c index 1c1c2c2..e04b61c 100644 --- a/tools/testing/selftests/bpf/btf_helpers.c +++ b/tools/testing/selftests/bpf/btf_helpers.c
@@ -27,11 +27,14 @@ static const char * const btf_kind_str_mapping[] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; static const char *btf_kind_str(__u16 kind) { - if (kind > BTF_KIND_ENUM64) + if (kind > BTF_KIND_LOCSEC) return "UNKNOWN"; return btf_kind_str_mapping[kind]; } @@ -111,9 +114,12 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_TYPEDEF: - case BTF_KIND_TYPE_TAG: fprintf(out, " type_id=%u", t->type); break; + case BTF_KIND_TYPE_TAG: + fprintf(out, " type_id=%u%s", t->type, + btf_kflag(t) ? " kflag=1" : ""); + break; case BTF_KIND_ARRAY: { const struct btf_array *arr = btf_array(t); @@ -200,9 +206,40 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) fprintf(out, " size=%u", t->size); break; case BTF_KIND_DECL_TAG: - fprintf(out, " type_id=%u component_idx=%d", - t->type, btf_decl_tag(t)->component_idx); + fprintf(out, " type_id=%u component_idx=%d%s", + t->type, btf_decl_tag(t)->component_idx, + btf_kflag(t) ? " kflag=1" : ""); break; + case BTF_KIND_LOC_PARAM: { + struct btf_loc_param *p = btf_loc_param(t); + + fprintf(out, " size=%u flags=0x%x vlen=%u", t->size, p->flags, vlen); + for (i = 0; i < vlen; i++) { + if (p->flags & BTF_LOC_PARAM_SIGNED) + fprintf(out, "\n\tvalue=%d", (__s32)p->values[i]); + else + fprintf(out, "\n\tvalue=%u", p->values[i]); + } + break; + } + case BTF_KIND_LOC_PROTO: { + const __u32 *p = btf_loc_proto_params(t); + + fprintf(out, " vlen=%u", vlen); + for (i = 0; i < vlen; i++, p++) + fprintf(out, "\n\ttype_id=%u", *p); + break; + } + case BTF_KIND_LOCSEC: { + const struct btf_loc *l = btf_locsec_locs(t); + + fprintf(out, " vlen=%u", vlen); + for (i = 0; i < vlen; i++, l++) { + fprintf(out, "\n\tfunc_type_id=%u loc_proto_type_id=%u offset=%u", + l->func, l->loc_proto, l->offset); + } + break; + } default: break; }
diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 45cd0b4..4183ff6 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c
@@ -188,6 +188,73 @@ int write_cgroup_file_parent(const char *relative_path, const char *file, return __write_cgroup_file(cgroup_path, file, buf); } +static int __read_cgroup_file(const char *cgroup_path, const char *file, + char *buf, size_t len) +{ + char file_path[PATH_MAX + 1]; + ssize_t got; + int fd; + + snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file); + fd = open(file_path, O_RDONLY); + if (fd < 0) { + log_err("Opening %s", file_path); + return 1; + } + + got = read(fd, buf, len - 1); + if (got < 0) { + log_err("Reading %s", file_path); + close(fd); + return 1; + } + buf[got] = '\0'; + close(fd); + return 0; +} + +/** + * read_cgroup_file() - Read from a cgroup file + * @relative_path: The cgroup path, relative to the workdir + * @file: The name of the file in cgroupfs to read from + * @buf: Buffer to read into, NUL-terminated on success + * @len: Size of @buf + * + * Read from a file in the given cgroup's directory. + * + * If successful, 0 is returned. + */ +int read_cgroup_file(const char *relative_path, const char *file, + char *buf, size_t len) +{ + char cgroup_path[PATH_MAX - 24]; + + format_cgroup_path(cgroup_path, relative_path); + return __read_cgroup_file(cgroup_path, file, buf, len); +} + +/** + * read_cgroup_file_parent() - Read from a cgroup file in the parent process + * workdir + * @relative_path: The cgroup path, relative to the parent process workdir + * @file: The name of the file in cgroupfs to read from + * @buf: Buffer to read into, NUL-terminated on success + * @len: Size of @buf + * + * Read from a file in the given cgroup's directory under the parent process + * workdir. + * + * If successful, 0 is returned. + */ +int read_cgroup_file_parent(const char *relative_path, const char *file, + char *buf, size_t len) +{ + char cgroup_path[PATH_MAX - 24]; + + format_parent_cgroup_path(cgroup_path, relative_path); + return __read_cgroup_file(cgroup_path, file, buf, len); +} + /** * setup_cgroup_environment() - Setup the cgroup environment *
diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index 3857304..d42d2e1 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h
@@ -15,6 +15,10 @@ int write_cgroup_file(const char *relative_path, const char *file, const char *buf); int write_cgroup_file_parent(const char *relative_path, const char *file, const char *buf); +int read_cgroup_file(const char *relative_path, const char *file, + char *buf, size_t len); +int read_cgroup_file_parent(const char *relative_path, const char *file, + char *buf, size_t len); int cgroup_setup_and_join(const char *relative_path); int get_root_cgroup(void); int create_and_get_cgroup(const char *relative_path);
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index ea7044f..49dfe93 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config
@@ -12,15 +12,18 @@ # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set CONFIG_CGROUP_BPF=y CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_MLDSA=y CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y CONFIG_CRYPTO_USER_API=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_USER_API_SKCIPHER=y -CONFIG_CRYPTO_SKCIPHER=y +CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_ECB=y CONFIG_CRYPTO_AES=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y +CONFIG_DEBUG_INFO_BTF_INLINE=y CONFIG_DEBUG_INFO_DWARF4=y CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y @@ -55,6 +58,7 @@ CONFIG_LIVEPATCH=y CONFIG_LWTUNNEL=y CONFIG_LWTUNNEL_BPF=y +CONFIG_MEMCG=y CONFIG_MODULE_SIG=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_UNLOAD=y @@ -121,7 +125,8 @@ CONFIG_SECURITY=y CONFIG_SECURITYFS=y CONFIG_SYN_COOKIES=y -CONFIG_TEST_BPF=m +CONFIG_TMPFS=y +CONFIG_TMPFS_XATTR=y CONFIG_UDMABUF=y CONFIG_USERFAULTFD=y CONFIG_VSOCKETS=y
diff --git a/tools/testing/selftests/bpf/gen_bpf_skel.sh b/tools/testing/selftests/bpf/gen_bpf_skel.sh new file mode 100755 index 0000000..60942f24 --- /dev/null +++ b/tools/testing/selftests/bpf/gen_bpf_skel.sh
@@ -0,0 +1,99 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Link BPF object file(s) with "bpftool gen object" and generate a +# skeleton (or light skeleton) header from the result. +# +# Usage: +# BPFTOOL=<bpftool> gen_bpf_skel.sh --name NAME --skel OUT +# [options] OBJ... +# +# --name NAME skeleton object name ("name NAME" for bpftool) +# --skel OUT output header ("foo.skel.h" or "foo.lskel.h") +# --subskel OUT also generate a subskeleton header into OUT +# --lskel generate a light skeleton (bpftool gen skeleton -L) +# --sign sign the (light) skeleton, with the key and +# certificate taken from $PRIVATE_KEY and +# $VERIFICATION_CERT +# +# The inputs are linked three times and the 2nd and 3rd results compared, +# as a regression test for the determinism of "bpftool gen object". +# +# Intermediate files are named after the output header with a "linked" +# infix - "llinked" for light or signed skeletons, so that generating a +# .skel.h and a .lskel.h from the same .bpf.o in parallel never races on +# the intermediates. +# +# The bpftool binary is taken from $BPFTOOL (default: bpftool from PATH). +# On failure all outputs and intermediates are removed and the script exits +# non-zero; permissive-mode skipping is the caller's business (see +# skip_on_fail in the Makefile). + +set -u + +bpftool=${BPFTOOL:-bpftool} +name='' skel='' subskel='' +lskel=0 sign=0 + +while [ $# -gt 0 ]; do + case "$1" in + --name) name=$2; shift 2 ;; + --skel) skel=$2; shift 2 ;; + --subskel) subskel=$2; shift 2 ;; + --lskel) lskel=1; shift ;; + --sign) sign=1; shift ;; + --) shift; break ;; + -*) echo "$0: unknown option: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +if [ -z "$name" ] || [ -z "$skel" ] || [ $# -eq 0 ]; then + echo "usage: $0 --name NAME --skel OUT [options] OBJ..." >&2 + exit 1 +fi + +infix=linked +if [ "$lskel" -eq 1 ] || [ "$sign" -eq 1 ]; then + infix=llinked +fi + +if [ "$sign" -eq 1 ] && { [ -z "${PRIVATE_KEY-}" ] || [ -z "${VERIFICATION_CERT-}" ]; }; then + echo "$0: --sign requires PRIVATE_KEY and VERIFICATION_CERT in the environment" >&2 + exit 1 +fi + +base=${skel%.skel.h} +base=${base%.lskel.h} +t1=$base.${infix}1.o +t2=$base.${infix}2.o +t3=$base.${infix}3.o + +fail() { + rm -f "$skel" ${subskel:+"$subskel"} "$t1" "$t2" "$t3" + exit 1 +} + +"$bpftool" gen object "$t1" "$@" || fail +"$bpftool" gen object "$t2" "$t1" || fail +"$bpftool" gen object "$t3" "$t2" || fail +if ! cmp -s "$t2" "$t3"; then + echo "$0: bpftool gen object is not deterministic for $skel" >&2 + fail +fi + +args=() +if [ "$sign" -eq 1 ]; then + args+=(-S -k "$PRIVATE_KEY" -i "$VERIFICATION_CERT") +fi +if [ "$lskel" -eq 1 ]; then + args+=(-L) +fi +"$bpftool" gen skeleton ${args[@]+"${args[@]}"} "$t3" name "$name" > "$skel" || fail + +if [ -n "$subskel" ]; then + "$bpftool" gen subskeleton "$t3" name "$name" > "$subskel" || fail +fi + +rm -f "$t1" "$t2" "$t3" +exit 0
diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile index 5e2ab51..91164d4 100644 --- a/tools/testing/selftests/bpf/libarena/Makefile +++ b/tools/testing/selftests/bpf/libarena/Makefile
@@ -27,10 +27,17 @@ INCLUDE_DIR ?= $(BPFDIR)/tools/include LIBBPF_INCLUDE ?= $(INCLUDE_DIR) -# Scan src/ and selftests/ to generate the final binaries -LIBARENA_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) $(wildcard $(LIBARENA)/selftests/*.bpf.c) -LIBARENA_OBJECTS = $(notdir $(LIBARENA_SOURCES:.bpf.c=.bpf.o)) -LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_SOURCES:.bpf.c=_asan.bpf.o)) +# Build selftests and benchmarks into separate BPF objects and skeletons. +LIBARENA_CORE_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) +LIBARENA_TEST_SOURCES = $(wildcard $(LIBARENA)/selftests/*.bpf.c) +LIBARENA_BENCH_SOURCES = $(wildcard $(LIBARENA)/benchs/*.bpf.c) + +LIBARENA_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=.bpf.o)) +LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=_asan.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=_asan.bpf.o)) +LIBARENA_BENCH_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_BENCH_SOURCES:.bpf.c=.bpf.o)) INCLUDES = -I$(LIBARENA)/include -I$(BPFDIR) ifneq ($(INCLUDE_DIR),) @@ -53,17 +60,25 @@ override BPF_CFLAGS += -Wno-incompatible-pointer-types-discards-qualifiers # Required for suppressing harmless vmlinux.h-related warnings. override BPF_CFLAGS += -Wno-missing-declarations +override BPF_CFLAGS += -fno-strict-aliasing override BPF_CFLAGS += $(INCLUDES) CFLAGS = -O2 -no-pie CFLAGS += $(INCLUDES) -vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests -vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests +vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs +vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs skeletons: libarena.skel.h libarena_asan.skel.h .PHONY: skeletons +benchmarks: libarena_bench.skel.h +.PHONY: benchmarks + +libarena_bench.skel.h: libarena_bench.bpf.o + $(call msg,GEN-SKEL,libarena,$@) + $(Q)$(BPFTOOL) gen skeleton $< name "libarena_bench" > $@ + libarena_asan.skel.h: libarena_asan.bpf.o $(call msg,GEN-SKEL,libarena,$@) $(Q)$(BPFTOOL) gen skeleton $< name "libarena_asan" > $@ @@ -80,6 +95,10 @@ $(call msg,GEN-OBJ,libarena,$@) $(Q)$(BPFTOOL) gen object $@ $^ +libarena_bench.bpf.o: $(LIBARENA_BENCH_OBJECTS) + $(call msg,GEN-OBJ,libarena,$@) + $(Q)$(BPFTOOL) gen object $@ $^ + %_asan.bpf.o: %.bpf.c $(call msg,CLNG-BPF,libarena,$@) $(Q)$(CLANG) $(BPF_CFLAGS) $(ASAN_FLAGS) -DBPF_ARENA_ASAN $(BPF_TARGET_ENDIAN) -c $< -o $@
diff --git a/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c new file mode 100644 index 0000000..e06397d --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c
@@ -0,0 +1,51 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/buddy.h> + +u32 bench_alloc_size; +u32 bench_nallocs; +long bench_hits; +long bench_duration_ns; + +SEC("syscall") +int bench_malloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_malloc(bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} + +SEC("syscall") +int bench_calloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_calloc(1, bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +}
diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index 71d9db6..055b589 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h
@@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ARENA_SPIN_LOCK_H #define BPF_ARENA_SPIN_LOCK_H
diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index 43c306e..65582b2 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h
@@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ATOMIC_H #define BPF_ATOMIC_H
diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h index 9ba9068..b32a420 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h
@@ -1,3 +1,4 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once /*
diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h index e2431ea..163e2b8 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h
@@ -1,5 +1,8 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once +#include <bpf_atomic.h> + #define BITS_PER_BYTE 8 #define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) @@ -15,11 +18,8 @@ struct arena_bitmap { struct arena_bitmap __arena *bmp_alloc(size_t bits); void bmp_free(struct arena_bitmap __arena *bmp); -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); @@ -32,3 +32,27 @@ void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); + +static __always_inline +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word |= BIT_MASK(bit); +} + +static __always_inline +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word &= ~BIT_MASK(bit); +} + +static __always_inline +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 word = READ_ONCE(bmp->bits[BIT_WORD(bit)]); + + return word & BIT_MASK(bit); +}
diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index 931ace9..a622035 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h
@@ -49,6 +49,7 @@ extern volatile u64 asan_violated; int arena_fls(__u64 word); void __arena *arena_malloc(size_t size); +void __arena *arena_calloc(size_t ncount, size_t size); void arena_free(void __arena *ptr); /* @@ -61,6 +62,76 @@ void arena_free(void __arena *ptr); */ #define arena_subprog_init() do { asm volatile ("" :: "r"(&arena)); } while (0) +/* + * BPF does not currently support the memset intrinsics. for large + * sequential copies, or assignments of large data structures, + * the frontend will generate an intrinsic that causes the BPF + * backend to exit due to a missing implementation. Provide + * implementations for the intrinsic. + */ +static inline int arena_memset(s8 __arena *dst, s8 val, size_t size) +{ + size_t headalign; + size_t tailalign; + u8 uval = (u8)val; + size_t val64; + size_t i; + + /* + * Calculate how many bytes to the next word-aligned one. + * We get this by truncating the 2s complement of the + * pointer to the last 3 bits. Intuitively, since + * + * The N LSBs of dst and -dst add to 1 << N, which + * is why dst + (-dst) = 0x0ULL through overflow. So the + * last N = 3 bits of the negative are the number of + * bytes to align dst on the last 3 bits. + * + */ + headalign = -(u64)dst & (sizeof(u64) - 1); + if (!headalign || size < headalign) + goto ptraligned; + + for (i = zero; i < headalign && can_loop; i++) + dst[i] = uval; + + dst += headalign; + size -= headalign; + +ptraligned: + + /* + * Make a word with all bytes equal to the byte we are setting. + * Since 1 byte -> 2 hex digits. + * + * Shifting the value by a 0 bytes is equal to multiplication by 0x01 + * Shifting by 1 bytes is equal to multiplication by 0x01 << 8, + * ... + * Shifting by 7 bytes is equal to multiplication by 0x01 << 56. + * + * End operation to replicate the byte into all the bytes of a word + * is (since a | b = a + b when a & b == 0): + * + * val + val * (1UL << 8) + val * (1UL << 16) + .. + val * (1UL << 56) + * = val * (1UL << 56 + 1UL << 48 + ... + 1UL << 0) + * = val * (0x01UL << 56 | 0x01UL << 48 + ... + 1UL << 0) + * = val * 0x0101 0101 0101 0101 + */ + val64 = (u8)val * 0x0101010101010101ULL; + + /* Pointer is now aligned, use word-aligned assignments. */ + for (i = zero; i < size / sizeof(u64) && can_loop; i++) + ((u64 __arena *)dst)[i] = val64; + + /* Go back to byte-aligned for the tail. */ + tailalign = size % sizeof(u64); + dst += size - tailalign; + for (i = zero; i < tailalign && can_loop; i++) + dst[i] = uval; + + return 0; +} + #else /* ! __BPF__ */ #include <stdint.h>
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c index 76319a5..e66b3a2 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c
@@ -1,3 +1,6 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + #include <libarena/common.h> #include <libarena/asan.h>
diff --git a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c index 5135d5c..de656b6 100644 --- a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c
@@ -103,23 +103,6 @@ volatile bool asan_inited = false; */ volatile bool asan_report_once = false; -/* - * BPF does not currently support the memset/memcpy/memcmp intrinsics. - * For large sequential copies, or assignments of large data structures, - * the frontend will generate an intrinsic that causes the BPF backend - * to exit due to a missing implementation. Provide a simple implementation - * just for memset to use it for poisoning/unpoisoning the map. - */ -__weak int asan_memset(s8 __arena *dst, s8 val, size_t size) -{ - size_t i; - - for (i = zero; i < size && can_loop; i++) - dst[i] = val; - - return 0; -} - /* Validate a 1-byte access, always within a single byte. */ static __always_inline bool memory_is_poisoned_1(s8 __arena *addr) { @@ -422,7 +405,7 @@ __hidden __noasan int asan_poison(void __arena *addr, s8 val, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, val, len); + arena_memset(shadow, val, len); return 0; } @@ -463,7 +446,7 @@ __hidden __noasan int asan_unpoison(void __arena *addr, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, 0, len); + arena_memset(shadow, 0, len); /* * If we are allocating a non-granule aligned region, we need to adjust
diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c index 5ff8e68..0390f20 100644 --- a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c
@@ -35,24 +35,6 @@ void bmp_free(struct arena_bitmap __arena *bmp) } __weak -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); -} - -__weak -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); -} - -__weak -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); -} - -__weak bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) { u64 val = BIT_MASK(bit);
diff --git a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c index c674ee5..2490ab1 100644 --- a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c
@@ -5,6 +5,8 @@ #include <libarena/asan.h> #include <libarena/buddy.h> +#include <bpf_arena_spin_lock.h> + /* * Buddy allocator arena-based implementation. * @@ -45,15 +47,8 @@ enum { BUDDY_CHUNK_PAGES = BUDDY_CHUNK_BYTES / __PAGE_SIZE }; -static inline int buddy_lock(struct buddy __arena *buddy) -{ - return arena_spin_lock(&buddy->lock); -} - -static inline void buddy_unlock(struct buddy __arena *buddy) -{ - arena_spin_unlock(&buddy->lock); -} +#define buddy_lock(buddy, flags) (arena_spin_lock_irqsave(&(buddy)->lock, (flags))) +#define buddy_unlock(buddy, flags) (arena_spin_unlock_irqrestore(&(buddy)->lock, (flags))) /* * Reserve part of the arena address space for the allocator. We use @@ -385,6 +380,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) { u64 order, ord, min_order, max_order; struct buddy_chunk __arena *chunk; + unsigned long flags; size_t left; int power2; u64 vaddr; @@ -416,7 +412,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) return NULL; } - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { /* * We cannot reclaim the vaddr space, but that is ok - this * operation should always succeed. The error path is to catch @@ -520,7 +516,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) arena_stderr( "chunk has size of 0x%lx bytes (left %lx bytes)\n", sizeof(*chunk), left); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -531,7 +527,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) order = (power2 >= BUDDY_MIN_ALLOC_SHIFT) ? power2 - BUDDY_MIN_ALLOC_SHIFT : 0; if (idx_set_allocated(chunk, idx, true)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -547,7 +543,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) */ min_order = left ? order + 1 : order; if (add_leftovers_to_freelist(chunk, idx, min_order, max_order)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -556,7 +552,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) max_order = order; } - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk; } @@ -564,6 +560,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) __weak int buddy_init(struct buddy __arena *buddy) { struct buddy_chunk __arena *chunk; + unsigned long flags; int ret; if (!asan_ready()) @@ -579,7 +576,7 @@ __weak int buddy_init(struct buddy __arena *buddy) chunk = buddy_chunk_get(buddy); - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { bpf_arena_free_pages(&arena, chunk, BUDDY_CHUNK_PAGES); return -EINVAL; } @@ -591,7 +588,7 @@ __weak int buddy_init(struct buddy __arena *buddy) /* Put the chunk at the beginning of the list. */ buddy->first_chunk = chunk; - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk ? 0 : -ENOMEM; } @@ -730,9 +727,10 @@ static u64 buddy_alloc_from_existing_chunks(struct buddy __arena *buddy, int ord */ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_chunk __arena *chunk, int order) { + unsigned long flags; u64 address; - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return (u64)NULL; @@ -745,7 +743,7 @@ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_ address = buddy_chunk_alloc(buddy->first_chunk, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return (u64)address; } @@ -754,6 +752,7 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) { void __arena *address = NULL; struct buddy_chunk __arena *chunk; + unsigned long flags; int order; if (!buddy) @@ -765,11 +764,11 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) return NULL; } - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return NULL; address = (u8 __arena *)buddy_alloc_from_existing_chunks(buddy, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); if (address) goto done; @@ -880,6 +879,7 @@ static __always_inline int buddy_free_unlocked(struct buddy __arena *buddy, u64 __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) { + unsigned long flags; int ret; if (!buddy) @@ -889,13 +889,13 @@ __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) if (!addr) return 0; - ret = buddy_lock(buddy); + ret = buddy_lock(buddy, flags); if (ret) return ret; ret = buddy_free_unlocked(buddy, (u64)addr); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return ret; }
diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 41b1de3..d1af60b 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c
@@ -1,5 +1,7 @@ // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <limits.h> + #include <libarena/common.h> #include <libarena/asan.h> #include <libarena/buddy.h> @@ -55,10 +57,33 @@ __weak void __arena *arena_malloc(size_t size) return buddy_alloc(&buddy, size); } +__weak void __arena *arena_calloc(size_t ncount, size_t size) +{ + void __arena *mem; + size_t total; + + /* + * Ideally we'd be using __builtin_mul_overflow here, + * but the BPF compiler backend doesn't implement __multi3. + * There are ways to optimize the division away from the + * overflow check, but any costs are dwarfed by the + * buddy_alloc() call. Keep it simple for now. + */ + if (unlikely(ncount && size >= ULLONG_MAX / ncount)) + return NULL; + + total = ncount * size; + + mem = buddy_alloc(&buddy, total); + if (likely(mem)) + arena_memset(mem, 0, total); + + return mem; +} + __weak void arena_free(void __arena *ptr) { buddy_free(&buddy, ptr); } - char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index cdf2d7d..d27dc79 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c
@@ -49,6 +49,8 @@ errno = __save; \ }) +#define CONNECT_MIN_TIMEOUT_MS 5000 + struct ipv4_packet pkt_v4 = { .eth.h_proto = __bpf_constant_htons(ETH_P_IP), .iph.ihl = 5, @@ -291,6 +293,37 @@ int client_socket(int family, int type, return -1; } +static int connect_with_timeout(int fd, const struct sockaddr_storage *addr, + socklen_t addrlen, int timeout_ms) +{ + int connect_timeout_ms = MAX(timeout_ms, CONNECT_MIN_TIMEOUT_MS); + + /* + * Override timeout configuration with a larger value for the + * connection + */ + if (settimeo(fd, connect_timeout_ms)) { + log_err("Failed to set connect timeout"); + return -1; + } + + if (connect(fd, (const struct sockaddr *)addr, addrlen)) { + log_err("Failed to connect"); + return -1; + } + + /* + * If the timeout configured by the test is different from the + * connect timeout, restore it + */ + if (timeout_ms != connect_timeout_ms && settimeo(fd, timeout_ms)) { + log_err("Failed to set timeout for connected socket"); + return -1; + } + + return 0; +} + int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t addrlen, const struct network_helper_opts *opts) { @@ -305,8 +338,7 @@ int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t add return -1; } - if (connect(fd, (const struct sockaddr *)addr, addrlen)) { - log_err("Failed to connect to server"); + if (connect_with_timeout(fd, addr, addrlen, opts->timeout_ms)) { save_errno_close(fd); return -1; } @@ -376,20 +408,12 @@ int connect_fd_to_fd(int client_fd, int server_fd, int timeout_ms) struct sockaddr_storage addr; socklen_t len = sizeof(addr); - if (settimeo(client_fd, timeout_ms)) - return -1; - if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) { log_err("Failed to get server addr"); return -1; } - if (connect(client_fd, (const struct sockaddr *)&addr, len)) { - log_err("Failed to connect to server"); - return -1; - } - - return 0; + return connect_with_timeout(client_fd, &addr, len, timeout_ms); } int make_sockaddr(int family, const char *addr_str, __u16 port,
diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 7513311..878c9fc 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h
@@ -129,12 +129,23 @@ static __u16 csum_fold(__u32 csum) static __wsum csum_partial(const void *buf, int len, __wsum sum) { - __u16 *p = (__u16 *)buf; + const __u8 *p = buf; int num_u16 = len >> 1; int i; for (i = 0; i < num_u16; i++) - sum += p[i]; + sum += ((const __u16 *)p)[i]; + + /* + * RFC 1071: an odd-length buffer's trailing byte is paired with + * a zero pad byte to form the final 16-bit word. + */ + if (len & 1) { + __u16 tail = 0; + + __builtin_memcpy(&tail, p + len - 1, 1); + sum += tail; + } return sum; }
diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c new file mode 100644 index 0000000..aa7562f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c
@@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "aggregate_arg_func.skel.h" +#include "aggregate_arg_kfunc.skel.h" + +void test_aggregate_arg(void) +{ + RUN_TESTS(aggregate_arg_func); + RUN_TESTS(aggregate_arg_kfunc); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c new file mode 100644 index 0000000..07d9d6e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c
@@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <bpf/btf.h> +#include "aggregate_ret_func.skel.h" +#include "aggregate_ret_kfunc.skel.h" +#include "aggregate_ret_kfunc_arena.skel.h" + +static bool testmod_has_arena_tagged_member(void) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_type *t; + bool tagged = false; + __s32 id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return false; + + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) + goto out; + + /* prog_test_ret_arena::a is 'void __arena_tag *': PTR -> TYPE_TAG -> void */ + id = btf__find_by_name_kind(module_btf, "prog_test_ret_arena", BTF_KIND_STRUCT); + if (id <= 0) + goto out; + + t = btf__type_by_id(module_btf, btf_members(btf__type_by_id(module_btf, id))[0].type); + if (!t || !btf_is_ptr(t)) + goto out; + + t = btf__type_by_id(module_btf, t->type); + tagged = t && btf_is_type_tag(t) && + !strcmp(btf__name_by_offset(module_btf, t->name_off), "arena"); + +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + + return tagged; +} + +void test_aggregate_ret(void) +{ + RUN_TESTS(aggregate_ret_func); + RUN_TESTS(aggregate_ret_kfunc); + + if (testmod_has_arena_tagged_member()) + RUN_TESTS(aggregate_ret_kfunc_arena); + else + test__skip(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index 1ad5d03..222c63b 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c
@@ -68,6 +68,12 @@ static void test_and(struct arena_atomics *skel) ASSERT_EQ(skel->arena->and64_value, 0x010ull << 32, "and64_value"); ASSERT_EQ(skel->arena->and32_value, 0x010, "and32_value"); + + ASSERT_EQ(skel->arena->and64_result, 0x110ull << 32, "and64_result"); + ASSERT_EQ(skel->arena->and32_result, 0x110, "and32_result"); + + ASSERT_EQ(skel->arena->and64_noreturn_value, 0x010ull << 32, "and64_noreturn_value"); + ASSERT_EQ(skel->arena->and32_noreturn_value, 0x010, "and32_noreturn_value"); } static void test_or(struct arena_atomics *skel) @@ -85,6 +91,12 @@ static void test_or(struct arena_atomics *skel) ASSERT_EQ(skel->arena->or64_value, 0x111ull << 32, "or64_value"); ASSERT_EQ(skel->arena->or32_value, 0x111, "or32_value"); + + ASSERT_EQ(skel->arena->or64_result, 0x110ull << 32, "or64_result"); + ASSERT_EQ(skel->arena->or32_result, 0x110, "or32_result"); + + ASSERT_EQ(skel->arena->or64_noreturn_value, 0x111ull << 32, "or64_noreturn_value"); + ASSERT_EQ(skel->arena->or32_noreturn_value, 0x111, "or32_noreturn_value"); } static void test_xor(struct arena_atomics *skel) @@ -102,6 +114,12 @@ static void test_xor(struct arena_atomics *skel) ASSERT_EQ(skel->arena->xor64_value, 0x101ull << 32, "xor64_value"); ASSERT_EQ(skel->arena->xor32_value, 0x101, "xor32_value"); + + ASSERT_EQ(skel->arena->xor64_result, 0x110ull << 32, "xor64_result"); + ASSERT_EQ(skel->arena->xor32_result, 0x110, "xor32_result"); + + ASSERT_EQ(skel->arena->xor64_noreturn_value, 0x101ull << 32, "xor64_noreturn_value"); + ASSERT_EQ(skel->arena->xor32_noreturn_value, 0x101, "xor32_noreturn_value"); } static void test_cmpxchg(struct arena_atomics *skel) @@ -146,6 +164,27 @@ static void test_xchg(struct arena_atomics *skel) ASSERT_EQ(skel->arena->xchg32_result, 1, "xchg32_result"); } +static void test_fetch_r0(struct arena_atomics *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + /* No need to attach it, just run it directly */ + prog_fd = bpf_program__fd(skel->progs.fetch_r0); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run_opts err")) + return; + if (!ASSERT_OK(topts.retval, "test_run_opts retval")) + return; + + ASSERT_EQ(skel->arena->fetch_src_r0_value, 0x11100000111ull, "fetch_src_r0_value"); + ASSERT_EQ(skel->arena->fetch_src_r0_result, 0x11000000110ull, "fetch_src_r0_result"); + + ASSERT_EQ(skel->arena->fetch_dst_r0_value, 0x11100000111ull, "fetch_dst_r0_value"); + ASSERT_EQ(skel->arena->fetch_dst_r0_result, 0x11000000110ull, "fetch_dst_r0_result"); + ASSERT_EQ(skel->arena->fetch_dst_r0_readback, 0x11100000111ull, "fetch_dst_r0_readback"); +} + static void test_uaf(struct arena_atomics *skel) { LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -160,6 +199,18 @@ static void test_uaf(struct arena_atomics *skel) return; ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails"); + +#ifndef __powerpc64__ + /* + * A fault clears the fetched register and leaves the address alone. The + * powerpc64 JIT does not clear the register of a faulting atomic yet. + */ + ASSERT_EQ(skel->arena->uaf_r1_after_dst_r0, 0, "uaf_r1_after_dst_r0"); + ASSERT_EQ(skel->arena->uaf_r0_after_dst_r0, skel->arena->uaf_r0_before_dst_r0, + "uaf_r0_after_dst_r0"); + ASSERT_EQ(skel->arena->uaf_r0_after_src_r0, 0, "uaf_r0_after_src_r0"); + ASSERT_EQ(skel->arena->uaf_reread, 0, "uaf_reread"); +#endif } static void test_load_acquire(struct arena_atomics *skel) @@ -256,6 +307,8 @@ void serial_test_arena_atomics(void) test_cmpxchg(skel); if (test__start_subtest("xchg")) test_xchg(skel); + if (test__start_subtest("fetch_r0")) + test_fetch_r0(skel); if (test__start_subtest("uaf")) test_uaf(skel); if (test__start_subtest("load_acquire"))
diff --git a/tools/testing/selftests/bpf/prog_tests/arena_memcg.c b/tools/testing/selftests/bpf/prog_tests/arena_memcg.c new file mode 100644 index 0000000..c76a7eb --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_memcg.c
@@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include <test_progs.h> +#include <fcntl.h> +#include <linux/magic.h> +#include <sys/vfs.h> +#include <sys/wait.h> +#include <unistd.h> +#include <sys/user.h> +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#define PAGE_SIZE getpagesize() +#endif + +#include "cgroup_helpers.h" +#include "arena_memcg.skel.h" + +#define CG_PATH "/arena_memcg" + +/* Reclaimable page cache the child builds up before it gets capped. */ +#define RECLAIMABLE (128 * 1024 * 1024) +/* Headroom left under memory.max, far less than the arena we fault in. */ +#define HEADROOM (8 * 1024 * 1024) +/* Arena to fault in; it only fits by reclaiming the page cache. */ +#define ARENA_FAULT (64 * 1024 * 1024) +/* Child exit code for "this environment cannot host the test". */ +#define CHILD_UNSUPPORTED 9 + +static void dump_memcg(void) +{ + char buf[512]; + + if (!read_cgroup_file(CG_PATH, "memory.max", buf, sizeof(buf))) + fprintf(stderr, "memory.max: %s", buf); + if (!read_cgroup_file(CG_PATH, "memory.peak", buf, sizeof(buf))) + fprintf(stderr, "memory.peak: %s", buf); + if (!read_cgroup_file(CG_PATH, "memory.events", buf, sizeof(buf))) + fprintf(stderr, "memory.events:\n%s", buf); +} + +/* + * Fill the page cache with @size bytes of clean, reclaimable pages by + * reading a sparse temp file, the way the cgroup selftests do. Returns the + * fd, which must stay open: closing it drops the cache. Returns -EOPNOTSUPP + * if the working directory cannot back such a file, -1 on error. + */ +static int alloc_pagecache(size_t size) +{ + struct statfs stfs; + char buf[4096]; + size_t off; + int fd; + + fd = open(".", O_TMPFILE | O_RDWR | O_EXCL, 0600); + if (fd < 0) + return errno == EOPNOTSUPP ? -EOPNOTSUPP : -1; + /* tmpfs hands out shmem pages, which are not reclaimable without swap */ + if (fstatfs(fd, &stfs) || stfs.f_type == TMPFS_MAGIC) { + close(fd); + return -EOPNOTSUPP; + } + if (ftruncate(fd, size)) + goto err; + for (off = 0; off < size; off += sizeof(buf)) + if (read(fd, buf, sizeof(buf)) < 0) + goto err; + return fd; +err: + close(fd); + return -1; +} + +void serial_test_arena_memcg(void) +{ + int cgroup_fd = -1, status, err; + const long ps = PAGE_SIZE; + char buf[64]; + pid_t pid; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + goto out; + + cgroup_fd = create_and_get_cgroup(CG_PATH); + if (!ASSERT_OK_FD(cgroup_fd, "create_and_get_cgroup")) + goto out; + + /* No memory controller -> nothing to test. */ + if (read_cgroup_file(CG_PATH, "memory.current", buf, sizeof(buf))) { + fprintf(stderr, "%s:SKIP:no memory controller or other env error\n", + __func__); + test__skip(); + goto out; + } + + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto out; + if (pid == 0) { + struct arena_memcg *cskel; + __u32 i, npages; + char *base; + size_t sz; + long cur; + int fd; + + /* + * Everything runs in the child: the arena vma is VM_DONTCOPY so + * it does not survive fork(), and only the child should be under + * the limit. The work dir belongs to the parent, so use the + * _parent() helpers; errors come back as an exit code, ASSERT_* + * does not reach the parent from here. + */ + + /* Step 1: join the memcg, so what follows is charged to it. */ + snprintf(buf, sizeof(buf), "%d", getpid()); + if (write_cgroup_file_parent(CG_PATH, "cgroup.procs", buf)) + _exit(2); + + /* + * Step 2: load the arena. A map is charged to whoever creates + * it, hence joining first. + */ + cskel = arena_memcg__open_and_load(); + if (!cskel) + _exit(3); + base = bpf_map__initial_value(cskel->maps.arena, &sz); + if (!base) + _exit(4); + npages = ARENA_FAULT / ps; + if (npages > bpf_map__max_entries(cskel->maps.arena)) + _exit(5); + + /* Step 3: make RECLAIMABLE bytes of clean page cache. */ + fd = alloc_pagecache(RECLAIMABLE); + if (fd == -EOPNOTSUPP) + _exit(CHILD_UNSUPPORTED); + if (fd < 0) + _exit(6); + + /* + * Step 4: set memory.max to what we use now plus HEADROOM. The + * page cache is already inside the limit, so only HEADROOM is + * left. + */ + if (read_cgroup_file_parent(CG_PATH, "memory.current", buf, sizeof(buf))) + _exit(7); + cur = strtol(buf, NULL, 10); + snprintf(buf, sizeof(buf), "%ld", cur + HEADROOM); + if (write_cgroup_file_parent(CG_PATH, "memory.max", buf)) + _exit(8); + + /* + * Step 5: fault ARENA_FAULT of arena in, much more than + * HEADROOM. Once it hits memory.max every further page has to + * come from reclaiming the page cache. With the fix the + * fault-in reclaims and all of it succeeds; without it the + * allocation cannot reclaim and we die on a valid address. + */ + for (i = 0; i < npages; i++) + base[(size_t)i * ps] = 1; + _exit(0); /* fd deliberately kept open until here */ + } + + if (!ASSERT_EQ(waitpid(pid, &status, 0), pid, "waitpid")) + goto out; + + /* The working directory cannot hold a reclaimable page cache. */ + if (WIFEXITED(status) && WEXITSTATUS(status) == CHILD_UNSUPPORTED) { + fprintf(stderr, "%s:SKIP:no disk-backed O_TMPFILE in cwd\n", __func__); + test__skip(); + goto out; + } + + /* A non-zero exit means the child failed to set up; the code says where. */ + if (WIFEXITED(status) && WEXITSTATUS(status)) { + ASSERT_OK(WEXITSTATUS(status), "child setup"); + goto out; + } + + /* + * With the fix the arena fault-in reclaims the page cache and every + * fault succeeds, so the child exits 0. Without it the allocation + * cannot reclaim, fails once the headroom is used up, and the child + * dies with SIGSEGV on a valid arena address. + */ + if (!ASSERT_TRUE(WIFEXITED(status) && !WEXITSTATUS(status), + "child faulted the arena in")) { + if (WIFSIGNALED(status)) + fprintf(stderr, "child killed by signal %d\n", WTERMSIG(status)); + dump_memcg(); + } +out: + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/arena_race.c b/tools/testing/selftests/bpf/prog_tests/arena_race.c new file mode 100644 index 0000000..f9d0e9c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_race.c
@@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <sched.h> +#include <setjmp.h> +#include <signal.h> +#include <sys/syscall.h> +#include <test_progs.h> + +#include "arena_race.skel.h" + +struct free_thread_ctx { + struct arena_race *skel; + int err; + __u32 retval; +}; + +struct fault_thread_ctx { + __u64 *addr; + int stop; +}; + +static int run_prog(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(err, name) && ASSERT_OK(opts.retval, name) ? 0 : -1; +} + +/* Trigger the sleepable free page path. */ +static void *run_free_thread(void *arg) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct free_thread_ctx *ctx = arg; + + ctx->skel->bss->target_tid = sys_gettid(); + ctx->err = bpf_prog_test_run_opts( + bpf_program__fd(ctx->skel->progs.free_page), &opts); + ctx->retval = opts.retval; + return NULL; +} + +static __thread sigjmp_buf reader_env; +static __thread bool reader_armed; + +/* Only the reader expects SIGBUS; anywhere else it keeps its default action. */ +static void reader_sigbus(int sig) +{ + if (!reader_armed) { + signal(sig, SIG_DFL); + raise(sig); + return; + } + siglongjmp(reader_env, 1); +} + +/* Continuously fault in the address. */ +static void *fault_reader_thread(void *arg) +{ + struct fault_thread_ctx *ctx = arg; + __u64 *addr; + + /* Touching a page while BPF frees it raises SIGBUS; skip that read. */ + sigsetjmp(reader_env, 1); + reader_armed = true; + while (!READ_ONCE(ctx->stop)) { + addr = __atomic_load_n(&ctx->addr, __ATOMIC_ACQUIRE); + (void)READ_ONCE(*addr); + } + reader_armed = false; + return NULL; +} + +static int wait_for(int *p) +{ + __u64 deadline = get_time_ns() + 5ULL * 1000 * 1000 * 1000; + + while (!READ_ONCE(*p)) { + if (get_time_ns() > deadline) + return -ETIMEDOUT; + } + return 0; +} + +/* Skip the subtest, saying why, if @func cannot be traced with fentry. */ +static bool require_fentry_target(const char *test, const char *func) +{ + if (libbpf_find_vmlinux_btf_id(func, BPF_TRACE_FENTRY) > 0) + return true; + printf("%s:SKIP: %s is not an fentry target\n", test, func); + test__skip(); + return false; +} + +static struct arena_race *setup_arena(__u64 **addr, bool trace_zap) +{ + struct arena_race *skel; + size_t arena_sz; + char *base; + int err; + + skel = arena_race__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return NULL; + err = bpf_program__set_autoload(skel->progs.trace_zap, trace_zap); + if (!ASSERT_OK(err, "trace_zap_autoload")) + goto err_out; + + err = arena_race__load(skel); + if (!ASSERT_OK(err, "load")) + goto err_out; + err = arena_race__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto err_out; + + if (run_prog(skel->progs.alloc_old, "alloc_old")) + goto err_out; + if (skel->bss->skip) { + printf("%s:SKIP: compiler doesn't support arena_cast\n", __func__); + test__skip(); + goto err_out; + } + + base = bpf_map__initial_value(skel->maps.arena, &arena_sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto err_out; + *addr = (__u64 *)(base + getpagesize()); + if (!ASSERT_EQ((unsigned long)skel->bss->ptr, (unsigned long)*addr, + "arena_ptr")) + goto err_out; + return skel; + +err_out: + arena_race__destroy(skel); + return NULL; +} + +static void test_realloc_during_free(bool deferred) +{ + struct free_thread_ctx ctx = {}; + struct arena_race *skel; + pthread_t thread; + __u64 *addr; + bool thread_created = false, zap_seen = false, completed = false; + int err; + + if (!require_fentry_target(__func__, "zap_vma_range")) + return; + + skel = setup_arena(&addr, true); + if (!skel) + return; + + /* Map the page into user space, so that the free has to zap it. */ + if (!ASSERT_EQ(READ_ONCE(*addr), skel->rodata->old_marker, "old_marker")) + goto out; + + /* Pause the free after its TLB flush, before it releases the range. */ + skel->bss->pause_on_zap = 1; + + if (deferred) { + /* + * Test the nonsleepable free path that gets + * deferred to a worker in the kernel. We do + * so by triggering the arena operation from + * a nonsleepable tracepoint context. + */ + skel->bss->trigger_pid_tgid = + ((__u64)getpid() << 32) | (__u32)sys_gettid(); + skel->bss->trigger_syscall = SYS_getpgid; + skel->bss->deferred_free = 1; + if (!ASSERT_GE(syscall(SYS_getpgid, 0), 0, "deferred_free")) + goto release; + } else { + /* + * Test the sleepable arena free path through a + * syscall test prog. + */ + ctx.skel = skel; + err = pthread_create(&thread, NULL, run_free_thread, &ctx); + if (!ASSERT_OK(err, "pthread_create")) { + skel->bss->release = 1; + goto out; + } + thread_created = true; + } + + /* Wait until the free reaches the user zap. */ + err = wait_for(&skel->bss->zap_entered); + if (!ASSERT_OK(err, "zap_entered")) + goto release; + + zap_seen = true; + + /* The range must stay allocated until the free completes. */ + run_prog(skel->progs.try_realloc, "realloc_during_free"); + ASSERT_NULL(skel->bss->realloc_ptr, "realloc_during_free"); + +release: + skel->bss->release = 1; + if (thread_created) { + ASSERT_OK(pthread_join(thread, NULL), "pthread_join"); + thread_created = false; + completed = ASSERT_OK(ctx.err, "free_run") && + ASSERT_OK(ctx.retval, "free_retval"); + } else if (skel->bss->target_tid) { + err = wait_for(&skel->bss->worker_exited); + completed = ASSERT_OK(err, "worker_exited"); + } + ASSERT_FALSE(skel->bss->timed_out, "zap_timed_out"); + + if (zap_seen && completed && + !run_prog(skel->progs.try_realloc, "realloc_after_free")) { + ASSERT_EQ((unsigned long)skel->bss->realloc_ptr, + (unsigned long)addr, "realloc_after_free"); + ASSERT_EQ(*addr, skel->rodata->new_marker, "new_marker"); + } +out: + arena_race__destroy(skel); +} + +/* + * Force a race between a faulting thread in userspace and a + * free operation on the arena. + */ +static void test_fault_free_realloc(void) +{ + struct sigaction sigbus = { .sa_handler = reader_sigbus }, old_sigbus; + struct fault_thread_ctx fault = {}; + struct arena_race *skel; + pthread_t fault_thread; + bool fault_created = false; + __u64 *addr; + __u64 expected, value; + int err, i; + + skel = setup_arena(&addr, false); + if (!skel) + return; + + skel->bss->realloc_after_free = 1; + + if (!ASSERT_OK(sigaction(SIGBUS, &sigbus, &old_sigbus), "sigaction_sigbus")) + goto out_destroy; + fault.addr = addr; + err = pthread_create(&fault_thread, NULL, fault_reader_thread, &fault); + if (!ASSERT_OK(err, "pthread_create_fault")) + goto out; + fault_created = true; + + for (i = 0; i < 1000; i++) { + LIBBPF_OPTS(bpf_test_run_opts, opts); + + /* Keep deferred frees from earlier iterations on separate slots. */ + if (i) { + skel->bss->page_idx = i; + if (run_prog(skel->progs.alloc_old, "alloc_next")) + goto out; + addr = (__u64 *)(unsigned long)skel->bss->ptr; + __atomic_store_n(&fault.addr, addr, __ATOMIC_RELEASE); + } + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.free_page), + &opts); + if (err) { + ASSERT_OK(err, "free_realloc"); + break; + } + if (opts.retval) { + ASSERT_OK(opts.retval, "free_realloc"); + break; + } + /* The reader may populate the slot first, or free may defer. */ + if (!skel->bss->realloc_ptr) + continue; + value = *addr; + expected = skel->bss->current_marker; + if (value != expected) { + ASSERT_EQ(value, expected, "marker_after_realloc"); + break; + } + } + + WRITE_ONCE(fault.stop, 1); + if (fault_created) { + ASSERT_OK(pthread_join(fault_thread, NULL), "pthread_join_fault"); + fault_created = false; + } +out: + WRITE_ONCE(fault.stop, 1); + if (fault_created) + pthread_join(fault_thread, NULL); + sigaction(SIGBUS, &old_sigbus, NULL); +out_destroy: + arena_race__destroy(skel); +} + +void serial_test_arena_race(void) +{ + cpu_set_t cpuset; + int err; + + err = sched_getaffinity(0, sizeof(cpuset), &cpuset); + if (!ASSERT_OK(err, "sched_getaffinity")) + return; + if (CPU_COUNT(&cpuset) < 2) { + printf("%s:SKIP: at least two runnable CPUs are required\n", __func__); + test__skip(); + return; + } + + if (test__start_subtest("realloc_during_free")) + test_realloc_during_free(false); + if (test__start_subtest("realloc_during_deferred_free")) + test_realloc_during_free(true); + if (test__start_subtest("fault_free_realloc")) + test_fault_free_realloc(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c new file mode 100644 index 0000000..2ac2e9a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c
@@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "sysctl_helpers.h" +#include "verifier_arena_scalar.skel.h" + +/* The same tests with constants of the programs blinded */ +void serial_test_arena_scalar_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + RUN_TESTS(verifier_arena_scalar); + sysctl_set_or_fail(harden, NULL, old); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index e8c1a61..41136e1 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
@@ -543,8 +543,8 @@ static void test_kprobe_sleepable(void) return; /* sleepable kprobe test case needs flags set before loading */ - if (!ASSERT_OK(bpf_program__set_flags(skel->progs.handle_kprobe_sleepable, - BPF_F_SLEEPABLE), "kprobe_sleepable_flags")) + if (!ASSERT_OK(bpf_program__add_flags(skel->progs.handle_kprobe_sleepable, BPF_F_SLEEPABLE), + "kprobe_sleepable_flags")) goto cleanup; if (!ASSERT_OK(test_attach_kprobe_sleepable__load(skel),
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 0222a9a..4ffe4fe 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c
@@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 #include <bpf/bpf.h> +#include <bpf/btf.h> #include <test_progs.h> -#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) static int map_create(__u32 map_type, __u32 max_entries) { const char *map_name = "insn_array"; @@ -453,43 +455,672 @@ static void check_bpf_no_lookup(void) close(map_fd); } +#define GOTOX_LOG_SZ (256 * 1024) + +/* Load and return the fd, which the caller owns. */ +static int gotox_prog_load_funcs_fd(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + + log[0] = 0; + opts.fd_array = fd_array; + opts.fd_array_cnt = fd_array_cnt; + opts.log_buf = log; + opts.log_size = GOTOX_LOG_SZ; + opts.log_level = 1; + if (fi_cnt) { + opts.prog_btf_fd = btf_fd; + opts.func_info = fi; + opts.func_info_cnt = fi_cnt; + opts.func_info_rec_size = sizeof(*fi); + } + + return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); +} + +/* Load for the outcome only: 0 or the negative errno. */ +static int gotox_prog_load_funcs(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) +{ + int prog_fd; + + prog_fd = gotox_prog_load_funcs_fd(insns, insn_cnt, fd_array, fd_array_cnt, log, + btf_fd, fi, fi_cnt); + if (prog_fd < 0) + return prog_fd; + close(prog_fd); + return 0; +} + +static int gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log) +{ + return gotox_prog_load_funcs(insns, insn_cnt, fd_array, fd_array_cnt, log, + -1, NULL, 0); +} + +static int gotox_jt_create_offs(const __u32 *offs, __u32 cnt) +{ + struct bpf_insn_array_value val = {}; + int map_fd; + __u32 i; + + map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, cnt); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return map_fd; + + for (i = 0; i < cnt; i++) { + val.orig_off = offs[i]; + if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, + "bpf_map_update_elem")) + goto err; + } + + if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) + goto err; + + return map_fd; +err: + close(map_fd); + return -1; +} + +#define GOTOX_SUB_START 4 +#define GOTOX_MAIN_TGT 2 +#define GOTOX_SUB_TGT 8 +#define GOTOX_TWO_INSN_CNT 10 + +static void gotox_two_subprogs_fill(struct bpf_insn *insns, __u32 jt_idx, __u32 jt_off) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_SUB_START - 1 - 1); + insns[GOTOX_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[3] = BPF_EXIT_INSN(); + + /* r1 = &jt[0], by index 'jt_idx' into fd_array */ + insns[GOTOX_SUB_START] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = jt_idx, + }; + insns[GOTOX_SUB_START + 1] = (struct bpf_insn) { .imm = 0 }; + insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, jt_off * 8); + insns[7] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[9] = BPF_EXIT_INSN(); +} + +/* + * Load 'insns' with the two jump tables 'jt_a' and 'jt_b', in this order in + * fd_array, and check the outcome. 'fi' describes the subprograms when given. + */ +static void check_gotox_two_maps(struct bpf_insn *insns, __u32 insn_cnt, + const __u32 *jt_a, __u32 a_cnt, + const __u32 *jt_b, __u32 b_cnt, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt, + int expect_err, const char *expect_msg) +{ + int map_fd[2] = { -1, -1 }; + char *log; + int err; + + log = calloc(1, GOTOX_LOG_SZ); + if (!ASSERT_OK_PTR(log, "calloc log")) + return; + + map_fd[0] = gotox_jt_create_offs(jt_a, a_cnt); + if (map_fd[0] < 0) + goto free_log; + map_fd[1] = gotox_jt_create_offs(jt_b, b_cnt); + if (map_fd[1] < 0) + goto close_maps; + + if (fi) + err = gotox_prog_load_funcs(insns, insn_cnt, map_fd, 2, log, + btf_fd, fi, fi_cnt); + else + err = gotox_prog_load(insns, insn_cnt, map_fd, 2, log); + if (expect_err) + ASSERT_EQ(err, expect_err, "program should have been rejected"); + else + ASSERT_EQ(err, 0, "bpf(BPF_PROG_LOAD)"); + if (expect_msg) + ASSERT_HAS_SUBSTR(log, expect_msg, "verifier log"); + +close_maps: + close(map_fd[0]); + if (map_fd[1] >= 0) + close(map_fd[1]); +free_log: + free(log); +} + +static void check_gotox_target_other_subprog(void) +{ + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + + gotox_two_subprogs_fill(insns, 0, 0); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); +} + +static void check_gotox_jt_per_subprog(void) +{ + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + + gotox_two_subprogs_fill(insns, 1, 0); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + 0, NULL); +} + +#define GOTOX_DEAD_MAIN_LEN 7 +#define GOTOX_DEAD_SUB_LEN 6 +#define GOTOX_DEAD_INSN_CNT (GOTOX_DEAD_MAIN_LEN + 3 * GOTOX_DEAD_SUB_LEN) +#define GOTOX_DEAD_LIVE_CNT 10 + +/* r1 = &jt[0], by index 'jt_idx' into fd_array; r1 = *r1; gotox r1; r0 = 0; exit */ +static void gotox_sub_fill(struct bpf_insn *insns, __u32 start, __u32 jt_idx) +{ + insns[start] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = jt_idx, + }; + insns[start + 1] = (struct bpf_insn) { .imm = 0 }; + insns[start + 2] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[start + 3] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[start + 4] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[start + 5] = BPF_EXIT_INSN(); +} + +/* + * Two subprograms which are only called from dead code sit in front of the + * live one, and all three own a jump table. Dead code removal drops both in + * one go, which compacts subprog_info[] along with the tables. + */ +static void check_gotox_jt_dead_subprogs(void) +{ + const __u32 sub_start[3] = { GOTOX_DEAD_MAIN_LEN, + GOTOX_DEAD_MAIN_LEN + GOTOX_DEAD_SUB_LEN, + GOTOX_DEAD_MAIN_LEN + 2 * GOTOX_DEAD_SUB_LEN }; + struct bpf_insn insns[GOTOX_DEAD_INSN_CNT]; + int map_fd[3] = { -1, -1, -1 }; + struct bpf_prog_info info = {}; + __u32 len = sizeof(info), tgt, i; + int prog_fd; + char *log; + + log = calloc(1, GOTOX_LOG_SZ); + if (!ASSERT_OK_PTR(log, "calloc log")) + return; + + insns[0] = BPF_MOV64_IMM(BPF_REG_1, 0); + insns[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2); + insns[2] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[0] - 2 - 1); + insns[3] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[1] - 3 - 1); + insns[4] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[2] - 4 - 1); + insns[5] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[6] = BPF_EXIT_INSN(); + + for (i = 0; i < ARRAY_SIZE(sub_start); i++) { + gotox_sub_fill(insns, sub_start[i], i); + tgt = sub_start[i] + 4; + map_fd[i] = gotox_jt_create_offs(&tgt, 1); + if (map_fd[i] < 0) + goto close_maps; + } + + prog_fd = gotox_prog_load_funcs_fd(insns, ARRAY_SIZE(insns), map_fd, + ARRAY_SIZE(map_fd), log, -1, NULL, 0); + if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) + goto close_maps; + + /* Both dead subprograms and the calls to them must be gone. */ + if (ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), + "bpf_prog_get_info_by_fd")) + ASSERT_EQ(info.xlated_prog_len / sizeof(struct bpf_insn), + GOTOX_DEAD_LIVE_CNT, "xlated insn count"); + close(prog_fd); +close_maps: + for (i = 0; i < ARRAY_SIZE(map_fd); i++) + if (map_fd[i] >= 0) + close(map_fd[i]); + free(log); +} + +#define GOTOX_FWD_GOTOX 11 +#define GOTOX_FWD_OWN_TGT 12 +#define GOTOX_FWD_SUB_START 14 +#define GOTOX_FWD_INSN_CNT 16 + +static void gotox_from_main_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_FWD_SUB_START - 1 - 1); + insns[2] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, + offsetof(struct xdp_md, ingress_ifindex)); + insns[3] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 4); + + /* r1 = &jt_leaves[0], by index 1 into fd_array */ + insns[4] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 1, + }; + insns[5] = (struct bpf_insn) { .imm = 0 }; + insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[7] = BPF_JMP_A(3); + + /* r1 = &jt_own[0], by index 0 into fd_array */ + insns[8] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[9] = (struct bpf_insn) { .imm = 0 }; + insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + + insns[GOTOX_FWD_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_FWD_OWN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[13] = BPF_EXIT_INSN(); + insns[GOTOX_FWD_SUB_START] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[15] = BPF_EXIT_INSN(); +} + +static void check_gotox_target_subprog_from_main(void) +{ + const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; + const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), + jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, + -EINVAL, "indirect jump from insn 11 to 14 leaves the subprog [0,14)"); +} + +/* + * The only map of the subprog holding the gotox reaches past that subprog, so + * the subprog is left without a jump table at all. + */ +static void check_gotox_jt_spans_subprogs(void) +{ + const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; + const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_span, ARRAY_SIZE(jt_span), + jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, + -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); +} + +/* + * The subprog holding the gotox has a well formed jump table of its own and + * also collects a map that reaches past its end. The spanning map is still + * rejected, even though the subprog is not left without a table. + */ +static void check_gotox_jt_spans_with_own_table(void) +{ + const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; + const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), + jt_span, ARRAY_SIZE(jt_span), -1, NULL, 0, + -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); +} + +#define GOTOX_SLICE_SUB_START 6 +#define GOTOX_SLICE_GOTOX 14 +#define GOTOX_SLICE_SUB_TGT 15 +#define GOTOX_SLICE_INSN_CNT 17 + +static void gotox_slice_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_SLICE_SUB_START - 1 - 1); + insns[2] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[3] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[4] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[5] = BPF_EXIT_INSN(); + + insns[GOTOX_SLICE_SUB_START] = + BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, + offsetof(struct xdp_md, ingress_ifindex)); + insns[7] = BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 1); + insns[8] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1); + insns[9] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + + /* r1 = &jt_main[0], by index 0 into fd_array */ + insns[10] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[11] = (struct bpf_insn) { .imm = 0 }; + insns[12] = BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_2); + insns[13] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + + insns[GOTOX_SLICE_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_SLICE_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[16] = BPF_EXIT_INSN(); +} + +static void check_gotox_index_slice_other_subprog(void) +{ + const __u32 jt_main[] = { 2, 3, 4 }; + const __u32 jt_sub[] = { GOTOX_SLICE_SUB_TGT }; + struct bpf_insn insns[GOTOX_SLICE_INSN_CNT]; + + gotox_slice_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + -EINVAL, "indirect jump from insn 14 to 3 leaves the subprog [6,17)"); +} + +static int gotox_btf_create(const __u32 *starts, const __u8 *linkage, __u32 cnt, + struct bpf_func_info *fi, struct btf **pbtf) +{ + int int_id, proto_id, id; + struct btf *btf; + char name[24]; + __u32 i; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return -1; + + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(int_id, 0, "btf__add_int")) + goto err; + + proto_id = btf__add_func_proto(btf, int_id); + if (!ASSERT_GT(proto_id, 0, "btf__add_func_proto")) + goto err; + + for (i = 0; i < cnt; i++) { + snprintf(name, sizeof(name), "gotox_f%u", i); + id = btf__add_func(btf, name, linkage[i], proto_id); + if (!ASSERT_GT(id, 0, "btf__add_func")) + goto err; + fi[i].insn_off = starts[i]; + fi[i].type_id = id; + } + + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto err; + + *pbtf = btf; + return btf__fd(btf); +err: + btf__free(btf); + return -1; +} + +static void check_gotox_target_other_global_subprog(void) +{ + const __u32 starts[] = { 0, GOTOX_SUB_START }; + const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_GLOBAL }; + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + struct bpf_func_info fi[2]; + struct btf *btf = NULL; + int btf_fd; + + gotox_two_subprogs_fill(insns, 0, 0); + + btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); + if (btf_fd < 0) + return; + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), btf_fd, fi, ARRAY_SIZE(fi), + -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); + btf__free(btf); +} + +#define GOTOX_CB_MAIN_TGT 6 +#define GOTOX_CB_START 8 +#define GOTOX_CB_GOTOX 11 +#define GOTOX_CB_TGT 12 +#define GOTOX_CB_INSN_CNT 14 + +static void gotox_callback_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_1, 1); + /* r2 = &callback */ + insns[1] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_2, + .src_reg = BPF_PSEUDO_FUNC, + .imm = GOTOX_CB_START - 1 - 1, + }; + insns[2] = (struct bpf_insn) { .imm = 0 }; + insns[3] = BPF_MOV64_IMM(BPF_REG_3, 0); + insns[4] = BPF_MOV64_IMM(BPF_REG_4, 0); + insns[5] = BPF_EMIT_CALL(BPF_FUNC_loop); + insns[GOTOX_CB_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[7] = BPF_EXIT_INSN(); + + /* r1 = &jt_main[0], by index 0 into fd_array */ + insns[GOTOX_CB_START] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[9] = (struct bpf_insn) { .imm = 0 }; + insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[GOTOX_CB_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_CB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[13] = BPF_EXIT_INSN(); +} + +static void check_gotox_callback_leaves_subprog(void) +{ + const __u32 starts[] = { 0, GOTOX_CB_START }; + const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_STATIC }; + const __u32 jt_main[] = { GOTOX_CB_MAIN_TGT }; + const __u32 jt_cb[] = { GOTOX_CB_TGT }; + struct bpf_insn insns[GOTOX_CB_INSN_CNT]; + struct bpf_func_info fi[2]; + struct btf *btf = NULL; + int btf_fd; + + gotox_callback_fill(insns); + + btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); + if (btf_fd < 0) + return; + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_cb, ARRAY_SIZE(jt_cb), btf_fd, fi, ARRAY_SIZE(fi), + -EINVAL, "indirect jump from insn 11 to 6 leaves the subprog [8,14)"); + btf__free(btf); +} + static void check_bpf_side(void) { check_bpf_no_lookup(); } +/* Return the xlated index of the first insn equal to 'insn', or -1. */ +static int xlated_insn_index(int prog_fd, const struct bpf_insn *insn) +{ + struct bpf_insn *xlated; + int idx = -1; + __u32 cnt, i; + + if (!ASSERT_OK(get_xlated_program(prog_fd, &xlated, &cnt), "get_xlated_program")) + return -1; + for (i = 0; i < cnt; i++) + if (!memcmp(&xlated[i], insn, sizeof(*insn))) { + idx = i; + break; + } + free(xlated); + return idx; +} + +/* + * Load 'insns' with a jump table naming insn 0, which a prologue prepend + * pushes down, and check that the slot followed it. + */ +static void check_gotox_target_prepend(enum bpf_prog_type prog_type, + struct bpf_insn *insns, __u32 insn_cnt) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + struct bpf_insn_array_value val = {}; + int prog_fd = -1, map_fd, idx; + __u32 key = 0; + + map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + val.orig_off = 0; + if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) + goto cleanup; + if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) + goto cleanup; + + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + prog_fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insn_cnt, &opts); + if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) + goto cleanup; + + idx = xlated_insn_index(prog_fd, &insns[0]); + if (!ASSERT_GT(idx, 0, "insn 0 was prepended to")) + goto cleanup; + if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &key, &val), 0, "bpf_map_lookup_elem")) + goto cleanup; + ASSERT_EQ(val.xlated_off, (__u32)idx, "gotox target followed insn 0 past the prepend"); +cleanup: + if (prog_fd >= 0) + close(prog_fd); + close(map_fd); +} + +static void check_gotox_target_prologue_shift(void) +{ + struct bpf_insn insns[] = { + /* insn 0: gotox target and subprog start */ + BPF_MOV64_IMM(BPF_REG_0, 0), + /* may_goto +4 -> exit block, bounds the loop */ + BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 4, 0), + /* r1 = &jt[0] (insns 2 and 3) */ + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0), /* r1 = ips[0] */ + /* insn 5: gotox r1 -> insn 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), /* insn 6: exit block */ + BPF_EXIT_INSN(), + }; + + check_gotox_target_prepend(BPF_PROG_TYPE_XDP, insns, ARRAY_SIZE(insns)); +} + +static void check_gotox_target_ctx_prologue_shift(void) +{ + struct bpf_insn insns[] = { + /* insn 0: gotox target and subprog start */ + BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, -8), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, 0x5a5a, 12), /* second visit exits */ + BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0x5a5a), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* r6 = ctx */ + BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, offsetof(struct __sk_buff, data)), + BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, offsetof(struct __sk_buff, data_end)), + BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 1), + BPF_JMP_REG(BPF_JGT, BPF_REG_4, BPF_REG_3, 5), + /* insn 9: direct packet write -> tc unclone prologue prepended at insn 0 */ + BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 0), + /* r5 = &jt[0] (insns 10 and 11) */ + BPF_LD_IMM64_RAW(BPF_REG_5, BPF_PSEUDO_MAP_IDX_VALUE, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_5, 0), /* r5 = ips[0] */ + /* insn 13: gotox r5 -> insn 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_5, 0, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), /* insn 14: exit block */ + BPF_EXIT_INSN(), + }; + + check_gotox_target_prepend(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns)); +} + static void __test_bpf_insn_array(void) { /* Test if offsets are adjusted properly */ - if (test__start_subtest("one2one")) check_one_to_one_mapping(); - if (test__start_subtest("simple")) check_simple(); - if (test__start_subtest("deletions")) check_deletions(); - if (test__start_subtest("deletions-with-functions")) check_deletions_with_functions(); - if (test__start_subtest("blindness")) check_blindness(); - /* Check all kinds of operations and related restrictions */ - if (test__start_subtest("incorrect-index")) check_incorrect_index(); - if (test__start_subtest("load-unfrozen-map")) check_load_unfrozen_map(); - if (test__start_subtest("no-map-reuse")) check_no_map_reuse(); - if (test__start_subtest("bpf-side-ops")) check_bpf_side(); + if (test__start_subtest("gotox-jt-spans-subprogs")) + check_gotox_jt_spans_subprogs(); + if (test__start_subtest("gotox-jt-spans-with-own-table")) + check_gotox_jt_spans_with_own_table(); + if (test__start_subtest("gotox-target-other-subprog")) + check_gotox_target_other_subprog(); + if (test__start_subtest("gotox-jt-per-subprog")) + check_gotox_jt_per_subprog(); + if (test__start_subtest("gotox-jt-dead-subprogs")) + check_gotox_jt_dead_subprogs(); + if (test__start_subtest("gotox-target-subprog-from-main")) + check_gotox_target_subprog_from_main(); + if (test__start_subtest("gotox-index-slice-other-subprog")) + check_gotox_index_slice_other_subprog(); + if (test__start_subtest("gotox-target-other-global-subprog")) + check_gotox_target_other_global_subprog(); + if (test__start_subtest("gotox-callback-leaves-subprog")) + check_gotox_callback_leaves_subprog(); + if (test__start_subtest("gotox-target-prologue-shift")) + check_gotox_target_prologue_shift(); + if (test__start_subtest("gotox-target-ctx-prologue-shift")) + check_gotox_target_ctx_prologue_shift(); } #else static void __test_bpf_insn_array(void)
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 14d4c17..d74a9db 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
@@ -13,13 +13,13 @@ struct { const char *prog_name; const char *err_msg; } test_bpf_nf_fail_tests[] = { - { "alloc_release", "kernel function bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, - { "insert_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, - { "lookup_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, - { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, - { "set_status_after_insert", "kernel function bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, - { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, - { "change_status_after_alloc", "kernel function bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, + { "alloc_release", "bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, + { "insert_insert", "bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "lookup_insert", "bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_timeout_after_insert", "bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_status_after_insert", "bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, + { "change_timeout_after_alloc", "bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, + { "change_status_after_alloc", "bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, { "write_not_allowlisted_field", "no write support to nf_conn at off" }, { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" },
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 6dbd148..122ecb7 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c
@@ -11,6 +11,7 @@ #include "bpf_qdisc_fail__invalid_dynptr.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_returned_slice.skel.h" #include "bpf_qdisc_fail__untrusted_write.skel.h" #include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" @@ -230,6 +231,7 @@ void test_ns_bpf_qdisc(void) test_incompl_ops(); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_returned_slice); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); RUN_TESTS(bpf_qdisc_fail__untrusted_write); RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone);
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c new file mode 100644 index 0000000..6435ed1c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c
@@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <network_helpers.h> +#include <bpf/btf.h> +#include "cgroup_helpers.h" +#include "bpf_tcp_ops.skel.h" + +#define CGROUP_PATH "/bpf_tcp_ops" +#define TEST_NETNS "bpf_tcp_ops" + +static __s32 get_bpf_tcp_ops_type_id(void) +{ + struct btf *vmlinux_btf; + __s32 type_id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "load_vmlinux_btf")) + return -1; + + type_id = btf__find_by_name_kind(vmlinux_btf, "bpf_tcp_ops", BTF_KIND_STRUCT); + btf__free(vmlinux_btf); + + ASSERT_GT(type_id, 0, "find_bpf_tcp_ops"); + return type_id; +} + +static void reset_order(struct bpf_tcp_ops *skel) +{ + memset(skel->bss->listen_order, 0, sizeof(skel->bss->listen_order)); + memset(skel->bss->connect_order, 0, sizeof(skel->bss->connect_order)); + skel->bss->listen_cnt = 0; + skel->bss->connect_cnt = 0; +} + +static void do_listen_connect(int family) +{ + const char *addr = family == AF_INET ? "127.0.0.1" : "::1"; + int server_fd, client_fd; + + server_fd = start_server(family, SOCK_STREAM, addr, 0, 0); + if (!ASSERT_GE(server_fd, 0, "start_server")) + return; + + client_fd = connect_to_fd(server_fd, 0); + if (ASSERT_OK_FD(client_fd, "connect_to_fd")) + close(client_fd); + + close(server_fd); +} + +/* + * Attach ops1 and ops2 normally (in that order), then ops3 with + * BPF_F_PREORDER. Expected execution order: [3, 1, 2] — ops3 runs + * first despite being attached last, ops1 before ops2 by attach order. + */ +static void test_order(int cgroup_fd, struct bpf_tcp_ops *skel, int family) +{ + LIBBPF_OPTS(bpf_cgroup_opts, preorder_opts, .flags = BPF_F_PREORDER); + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_ops2")) + goto done; + + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops3, cgroup_fd, + &preorder_opts); + if (!ASSERT_OK_PTR(link3, "attach_ops3_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(family); + + ASSERT_EQ(skel->bss->listen_cnt, 3, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 3, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "listen_order[1]"); + ASSERT_EQ(skel->bss->listen_order[2], 2, "listen_order[2]"); + + ASSERT_EQ(skel->bss->connect_cnt, 3, "connect_cnt"); + ASSERT_EQ(skel->bss->connect_order[0], 3, "connect_order[0]"); + ASSERT_EQ(skel->bss->connect_order[1], 1, "connect_order[1]"); + ASSERT_EQ(skel->bss->connect_order[2], 2, "connect_order[2]"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_order_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_order(cgroup_fd, skel, AF_INET); + test_order(cgroup_fd, skel, AF_INET6); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * Position a new attachment relative to an existing one. Attach ops1, then + * ops2 with BPF_F_BEFORE ops1, then ops3 with BPF_F_AFTER ops2. Expected + * execution order: [2, 3, 1]. For struct_ops, relative_fd refers to a link + * fd, so BPF_F_LINK must be set. + */ +static void test_before_after(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_cgroup_opts, opts); + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + opts.flags = BPF_F_BEFORE | BPF_F_LINK; + opts.relative_fd = bpf_link__fd(link1); + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, &opts); + if (!ASSERT_OK_PTR(link2, "attach_ops2_before")) + goto done; + + opts.flags = BPF_F_AFTER | BPF_F_LINK; + opts.relative_fd = bpf_link__fd(link2); + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops3, cgroup_fd, &opts); + if (!ASSERT_OK_PTR(link3, "attach_ops3_after")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 3, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "listen_order[1]"); + ASSERT_EQ(skel->bss->listen_order[2], 1, "listen_order[2]"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_before_after_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_before_after(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +static void test_query(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_prog_query_opts, query_opts); + struct bpf_map_info info = {}; + __u32 info_len = sizeof(info); + struct bpf_link *link1 = NULL, *link2 = NULL; + __u32 map1_id, map2_id, map_ids[2] = {}; + __s32 type_id; + + type_id = get_bpf_tcp_ops_type_id(); + if (type_id <= 0) + return; + + bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.tcp_ops1), &info, &info_len); + map1_id = info.id; + + bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.tcp_ops2), &info, &info_len); + map2_id = info.id; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_ops2")) + goto done; + + /* query effective: expect 2 entries in attachment order */ + query_opts.type_id = type_id; + query_opts.prog_ids = map_ids; + query_opts.count = ARRAY_SIZE(map_ids); + query_opts.query_flags = BPF_F_QUERY_EFFECTIVE; + ASSERT_OK(bpf_prog_query_opts(cgroup_fd, BPF_STRUCT_OPS, &query_opts), + "query_effective"); + ASSERT_EQ(query_opts.count, 2, "query_effective_count"); + ASSERT_EQ(map_ids[0], map1_id, "map_ids[0]"); + ASSERT_EQ(map_ids[1], map2_id, "map_ids[1]"); + + /* query attached (non-effective): expect 2 entries */ + memset(map_ids, 0, sizeof(map_ids)); + query_opts.query_flags = 0; + query_opts.count = ARRAY_SIZE(map_ids); + ASSERT_OK(bpf_prog_query_opts(cgroup_fd, BPF_STRUCT_OPS, &query_opts), + "query_attached"); + ASSERT_EQ(query_opts.count, 2, "query_attached_count"); + ASSERT_EQ(map_ids[0], map1_id, "attached_map_ids[0]"); + ASSERT_EQ(map_ids[1], map2_id, "attached_map_ids[1]"); + +done: + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_query_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_query(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* Must match progs/bpf_tcp_ops.c */ +#define OPS_RETVAL1 11 +#define OPS_RETVAL2 22 + +/* + * Attach three struct_ops implementing timeout_init to the same cgroup; they + * run in attach order [retval1, retval2, retval3]. timeout_init's return value + * is chained: the first prog reads the kernel seed via bpf_get_retval() (0, + * since no legacy sockops prog is attached) and returns OPS_RETVAL1; each + * subsequent prog must then observe the previous prog's return value. This + * proves the trampoline inherits the retval across an array of struct_ops. + */ +static void test_retval(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + skel->bss->retval_saw1 = -1; + skel->bss->retval_saw2 = -1; + skel->bss->retval_saw3 = -1; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_retval1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_retval2")) + goto done; + + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval3, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link3, "attach_retval3")) + goto done; + + do_listen_connect(AF_INET6); + + /* First prog inherits the kernel seed (no legacy sockops -> 0). */ + ASSERT_EQ(skel->bss->retval_saw1, 0, "retval_saw1"); + /* Each subsequent prog inherits the previous prog's return value. */ + ASSERT_EQ(skel->bss->retval_saw2, OPS_RETVAL1, "retval_saw2"); + ASSERT_EQ(skel->bss->retval_saw3, OPS_RETVAL2, "retval_saw3"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_retval_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_retval(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * bpf_link__update_map() swaps the struct_ops map backing an attached link. + * The link keeps its position, including BPF_F_PREORDER, across the update. + * Attach ops1 (normal) and ops2 (preorder): order [2, 1]. Update the normal + * link to ops3 -> [2, 3]; update the preorder link to ops1 -> [1, 3]. + */ +static void test_update(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_cgroup_opts, preorder_opts, .flags = BPF_F_PREORDER); + struct bpf_link *link = NULL, *link_pre = NULL; + + link = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link, "attach_ops1")) + goto done; + + link_pre = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, + &preorder_opts); + if (!ASSERT_OK_PTR(link_pre, "attach_ops2_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_cnt, 2, "cnt_initial"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "order0_initial"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "order1_initial"); + + /* Update the normal link's map (ops1 -> ops3); position is unchanged. */ + if (!ASSERT_OK(bpf_link__update_map(link, skel->maps.tcp_ops3), "update_normal")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_order[0], 2, "order0_after_normal"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "order1_after_normal"); + + /* Update the preorder link's map (ops2 -> ops1); it stays first. */ + if (!ASSERT_OK(bpf_link__update_map(link_pre, skel->maps.tcp_ops1), "update_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_order[0], 1, "order0_after_preorder"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "order1_after_preorder"); + +done: + bpf_link__destroy(link_pre); + bpf_link__destroy(link); +} + +static void run_update_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_update(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * Two-level hierarchy. Attach ops1 to the parent and ops2 to the child, then + * trigger from a socket in the child. Descendant progs run before ancestor + * progs, so the order is [2 (child), 1 (parent)]. + */ +static void test_hierarchy(int parent_fd, int child_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *plink = NULL, *clink = NULL; + + plink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, parent_fd, NULL); + if (!ASSERT_OK_PTR(plink, "attach_parent")) + goto done; + + clink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, child_fd, NULL); + if (!ASSERT_OK_PTR(clink, "attach_child")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 2, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "listen_order[1]"); + +done: + bpf_link__destroy(clink); + bpf_link__destroy(plink); +} + +static void run_hierarchy_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int parent_fd, child_fd = -1; + + parent_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(parent_fd, 0, "join_parent_cgroup")) + return; + + child_fd = create_and_get_cgroup(CGROUP_PATH "/child"); + if (!ASSERT_GE(child_fd, 0, "create_child_cgroup")) + goto done; + + if (!ASSERT_OK(join_cgroup(CGROUP_PATH "/child"), "join_child_cgroup")) + goto done; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_hierarchy(parent_fd, child_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + if (child_fd >= 0) { + close(child_fd); + join_cgroup(CGROUP_PATH); + remove_cgroup(CGROUP_PATH "/child"); + } + close(parent_fd); +} + +/* + * Attach ops1 to the parent, then create and join the child cgroup. The child + * is created after the attach, so it must inherit the parent's effective progs + * via cgroup_bpf_inherit(). A socket in the child runs the parent's prog. + */ +static void test_inherit(int parent_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *plink = NULL; + int child_fd = -1; + + plink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, parent_fd, NULL); + if (!ASSERT_OK_PTR(plink, "attach_parent")) + goto done; + + child_fd = create_and_get_cgroup(CGROUP_PATH "/child"); + if (!ASSERT_GE(child_fd, 0, "create_child_cgroup")) + goto done; + + if (!ASSERT_OK(join_cgroup(CGROUP_PATH "/child"), "join_child_cgroup")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 1, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 1, "listen_order[0]"); + +done: + if (child_fd >= 0) { + close(child_fd); + join_cgroup(CGROUP_PATH); + remove_cgroup(CGROUP_PATH "/child"); + } + bpf_link__destroy(plink); +} + +static void run_inherit_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int parent_fd; + + parent_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(parent_fd, 0, "join_parent_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_inherit(parent_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(parent_fd); +} + +void test_bpf_tcp_ops(void) +{ + if (test__start_subtest("order")) + run_order_subtest(); + if (test__start_subtest("before_after")) + run_before_after_subtest(); + if (test__start_subtest("query")) + run_query_subtest(); + if (test__start_subtest("retval")) + run_retval_subtest(); + if (test__start_subtest("update")) + run_update_subtest(); + if (test__start_subtest("hierarchy")) + run_hierarchy_subtest(); + if (test__start_subtest("inherit")) + run_inherit_subtest(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c new file mode 100644 index 0000000..244605e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c
@@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <network_helpers.h> +#include "cgroup_helpers.h" +#include "bpf_tcp_ops_hdr.skel.h" + +#define CGROUP_PATH "/bpf_tcp_ops_hdr" +#define TEST_NETNS "bpf_tcp_ops_hdr" + +#define TEST_OPT_D0 0xAB +#define TEST_OPT_D1 0xCD + +static void run_hdr_opt(void) +{ + struct bpf_tcp_ops_hdr *skel = NULL; + struct bpf_link *link = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd, lfd = -1, fd = -1; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops_hdr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + link = bpf_map__attach_cgroup_opts(skel->maps.test_hdr_ops, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link, "attach_cgroup")) + goto done; + + /* + * One direction of data is enough to exercise all hooks: both peers + * share the cgroup struct_ops, so the sender runs hdr_opt_len/write + * and the receiver runs parse. + */ + lfd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + if (!ASSERT_GE(lfd, 0, "start_server")) + goto done; + + fd = connect_to_fd(lfd, 0); + if (!ASSERT_OK_FD(fd, "connect_to_fd")) + goto done; + + if (!ASSERT_OK(send_recv_data(lfd, fd, 64), "send_recv_data")) + goto done; + + /* Reserve + write hooks ran while sending. */ + ASSERT_GT(skel->bss->hdr_opt_len_cnt, 0, "hdr_opt_len_cnt"); + ASSERT_GT(skel->bss->write_cnt, 0, "write_cnt"); + /* Parse hook ran and recovered our option on the receive side. */ + ASSERT_GT(skel->bss->parse_cnt, 0, "parse_cnt"); + ASSERT_GT(skel->bss->found_cnt, 0, "found_cnt"); + ASSERT_EQ(skel->bss->found_d0, TEST_OPT_D0, "found_d0"); + ASSERT_EQ(skel->bss->found_d1, TEST_OPT_D1, "found_d1"); + +done: + if (fd >= 0) + close(fd); + if (lfd >= 0) + close(lfd); + bpf_link__destroy(link); + bpf_tcp_ops_hdr__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +void test_bpf_tcp_ops_hdr(void) +{ + run_hdr_opt(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index 73f6690..902a70d 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
@@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type) } bpf_program__set_type(prog, type); - bpf_program__set_flags(prog, testing_prog_flags()); + bpf_program__add_flags(prog, testing_prog_flags()); bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags); err = bpf_object__load(obj);
diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c new file mode 100644 index 0000000..534aa79 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c
@@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpftool_helpers.h> +#include <bpf/btf.h> +#include <limits.h> +#include <unistd.h> +#include "btf_helpers.h" +#include "testing_helpers.h" + +#define DUMP_BUF_SZ (16 * 1024) + +#define EXPECTED_SORTED "bpftool_btf_dump_sorted.expected" +#define EXPECTED_UNSORTED "bpftool_btf_dump_unsorted.expected" + +/* + * struct holey { + * int c; + * <64-bit hole> + * int tail; + * }; + * enum { E0 = 1 }; + * struct s { int f; }; + */ +static struct btf *mk_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_int(btf, "long int", 4, BTF_INT_SIGNED); + + btf__add_struct(btf, "holey", 16); + btf__add_field(btf, "c", 1, 0, 0); + btf__add_field(btf, "tail", 1, 96, 0); + + btf__add_enum(btf, NULL, 4); + btf__add_enum_value(btf, "E0", 1); + + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'long int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[3] STRUCT 'holey' size=16 vlen=2\n" + "\t'c' type_id=1 bits_offset=0\n" + "\t'tail' type_id=1 bits_offset=96", + "[4] ENUM '(anon)' encoding=UNSIGNED size=4 vlen=1\n" + "\t'E0' val=1", + "[5] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0"); + + return btf; +} + +static struct btf *mk_loc_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_func_proto(btf, 1); + btf__add_func_param(btf, "arg1", 1); + btf__add_func_param(btf, "arg2", 1); + btf__add_func_param(btf, "arg3", 1); + btf__add_func_param(btf, "arg4", 1); + btf__add_func_param(btf, "arg5", 1); + btf__add_func_param(btf, "arg6", 1); + btf__add_func(btf, "foo", BTF_FUNC_STATIC, 2); + + btf__add_loc_param(btf, 4, BTF_LOC_PARAM_REG); + btf__add_loc_param_value(btf, 1); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET); + btf__add_loc_param_value(btf, 2); + btf__add_loc_param_value(btf, 0x10); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED); + btf__add_loc_param_value(btf, BTF_LOC_PARAM_FBREG); + btf__add_loc_param_value(btf, -0x10); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf, 0x9abcdef0); + btf__add_loc_param_value(btf, 0x12345678); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf, 0xfeedface); + btf__add_loc_param_value(btf, 0xdeadbeef); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED); + btf__add_loc_param_value(btf, BTF_LOC_PARAM_FBREG); + btf__add_loc_param_value(btf, -0x20); + btf__add_loc_proto(btf); + btf__add_loc_proto_param(btf, 4); + btf__add_loc_proto_param(btf, 5); + btf__add_loc_proto_param(btf, 6); + btf__add_loc_proto_param(btf, 7); + btf__add_loc_proto_param(btf, 8); + btf__add_loc_proto_param(btf, 9); + btf__add_locsec(btf, "inline.text"); + btf__add_locsec_loc(btf, 3, 10, 64); + + return btf; +} + +static int btf_to_tmpfile(const struct btf *btf, char *path) +{ + ssize_t written; + const void *raw; + __u32 sz; + int fd; + + raw = btf__raw_data(btf, &sz); + if (!ASSERT_OK_PTR(raw, "raw_data")) + return -1; + + snprintf(path, PATH_MAX, "/tmp/bpftool_btf_dump.XXXXXX"); + fd = mkstemp(path); + if (!ASSERT_OK_FD(fd, "mkstemp_btf")) + return -1; + + written = write(fd, raw, sz); + close(fd); + if (!ASSERT_EQ(written, sz, "write_btf")) { + unlink(path); + return -1; + } + + return 0; +} + +static char *dump_c(const char *btf_path, bool sorted) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s format c%s", + btf_path, sorted ? "" : " unsorted"); + + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_format_c")) { + free(buf); + return NULL; + } + + return buf; +} + +static char *dump_raw(const char *btf_path) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s", btf_path); + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_raw")) { + free(buf); + return NULL; + } + + return buf; +} + +static char *read_expected(const char *path) +{ + char *buf = NULL; + size_t cap = 0; + FILE *f; + int err; + + f = fopen(path, "r"); + if (!f) { + err = errno; + PRINT_FAIL("can't open expected output '%s': errno %d\n", path, err); + return NULL; + } + + /* no NUL in a generated header, so this reads to the end */ + if (getdelim(&buf, &cap, '\0', f) < 0) { + err = errno; + PRINT_FAIL("can't read expected output '%s': errno %d\n", path, err); + free(buf); + buf = NULL; + } + + fclose(f); + return buf; +} + +static void test_dump(const char *btf_path, bool sorted) +{ + const char *exp_path; + char *dump, *exp; + + exp_path = sorted ? EXPECTED_SORTED : EXPECTED_UNSORTED; + + dump = dump_c(btf_path, sorted); + if (!dump) + return; + + exp = read_expected(exp_path); + if (!exp) + goto out_dump; + + ASSERT_TEXT_EQ(dump, exp, sorted ? "cmp_sorted" : "cmp_unsorted"); + + free(exp); +out_dump: + free(dump); +} + +static void test_loc_dump(const char *btf_path) +{ + const char expected[] = + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED\n" + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=6\n" + "\t'arg1' type_id=1\n" + "\t'arg2' type_id=1\n" + "\t'arg3' type_id=1\n" + "\t'arg4' type_id=1\n" + "\t'arg5' type_id=1\n" + "\t'arg6' type_id=1\n" + "[3] FUNC 'foo' type_id=2 linkage=static\n" + "[4] LOC_PARAM '(anon)' size=4 flags=0x8 vlen=1 values='reg1'\n" + "[5] LOC_PARAM '(anon)' size=8 flags=0x38 vlen=2 values='*(reg2 + 0x10)'\n" + "[6] LOC_PARAM '(anon)' size=8 flags=0x29 vlen=2 values='fbreg - 0x10'\n" + "[7] LOC_PARAM '(anon)' size=8 flags=0x6 vlen=2 values='address 0x123456789abcdef0'\n" + "[8] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2 values='const 0xdeadbeeffeedface'\n" + "[9] LOC_PARAM '(anon)' size=8 flags=0x39 vlen=2 values='*(fbreg - 0x20)'\n" + "[10] LOC_PROTO '(anon)' vlen=6\n" + "\ttype_id=4 value='reg1'\n" + "\ttype_id=5 value='*(reg2 + 0x10)'\n" + "\ttype_id=6 value='fbreg - 0x10'\n" + "\ttype_id=7 value='address 0x123456789abcdef0'\n" + "\ttype_id=8 value='const 0xdeadbeeffeedface'\n" + "\ttype_id=9 value='*(fbreg - 0x20)'\n" + "[11] LOCSEC 'inline.text' vlen=1\n" + "\tfunc='foo(arg1 [reg1], arg2 [*(reg2 + 0x10)], arg3 [fbreg - 0x10], arg4 [address 0x123456789abcdef0], arg5 [const 0xdeadbeeffeedface], arg6 [*(fbreg - 0x20)])' func_type_id=3 loc_proto_type_id=10 offset=64\n"; + char *dump; + + dump = dump_raw(btf_path); + if (!dump) + return; + + ASSERT_TEXT_EQ(dump, expected, "cmp_loc_dump"); + free(dump); +} + +void test_bpftool_btf_dump(void) +{ + char path[PATH_MAX]; + struct btf *btf = NULL; + + btf = mk_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("c_sorted")) + test_dump(path, true); + if (test__start_subtest("c_unsorted")) + test_dump(path, false); + unlink(path); + + btf__free(btf); + + btf = mk_loc_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("loc_dump")) + test_loc_dump(path); + unlink(path); +out_btf: + btf__free(btf); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c new file mode 100644 index 0000000..187b3f7 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c
@@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <sys/socket.h> +#include <unistd.h> +#include <bpf/btf.h> +#include <test_progs.h> +#include "bpftool_helpers.h" + +#define PIN_PATH "/sys/fs/bpf/test_bpftool_map_dump_sk_storage" +#define QUEUE_PIN_PATH "/sys/fs/bpf/test_bpftool_map_dump_queue" +#define JSON_ERROR "{\"error\":\"can't dump sk_storage map" + +/* An empty socket storage map with an int value. */ +static int create_sk_storage_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NO_PREALLOC); + struct btf *btf; + int int_id, fd = -1; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return -1; + + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(int_id, 0, "btf__add_int")) + goto out; + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_key_type_id = int_id; + opts.btf_value_type_id = int_id; + fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage", sizeof(int), + sizeof(int), 0, &opts); + ASSERT_OK_FD(fd, "bpf_map_create"); +out: + btf__free(btf); + return fd; +} + +static void test_sk_storage(void) +{ + int map_fd, sk_fd = -1, value = 42, ret; + char output[1024] = {}; + + map_fd = create_sk_storage_map(); + if (map_fd < 0) + return; + + sk_fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(sk_fd, "socket")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST), + "add socket storage")) + goto out; + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin map")) + goto out; + + /* + * Socket storage can't be iterated, so the dump must fail with an + * error and nothing else, rather than print an empty map that holds + * an element. The buffer is cleared before each run so that a run + * that fails to start can't match stale or uninitialised output. + */ + ret = get_bpftool_command_output("-j map dump pinned " PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "json dump fails"); + ASSERT_STRNEQ(output, JSON_ERROR, strlen(JSON_ERROR), + "json dump is only the error"); + + memset(output, 0, sizeof(output)); + ret = get_bpftool_command_output("map dump pinned " PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "plain dump fails"); + ASSERT_NULL(strchr(output, '['), "plain dump prints no elements"); + + memset(output, 0, sizeof(output)); + get_bpftool_command_output("map dump pinned " PIN_PATH " 2>&1 >/dev/null", + output, sizeof(output)); + ASSERT_HAS_SUBSTR(output, + "can't dump sk_storage map: Operation not supported", + "plain dump explains why on stderr"); + +out: + unlink(PIN_PATH); + if (sk_fd >= 0) + close(sk_fd); + close(map_fd); +} + +/* + * A queue map has no BTF, so its plain dump prints an element count + * rather than JSON. A dump that ignored the error would print "Found 0 + * elements" for a queue that holds one. + */ +static void test_queue(void) +{ + char output[1024] = {}; + int map_fd, value = 42, ret; + + map_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, "queue", 0, sizeof(int), 1, + NULL); + if (!ASSERT_OK_FD(map_fd, "bpf_map_create")) + return; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, NULL, &value, BPF_ANY), + "push to queue")) + goto out; + if (!ASSERT_OK(bpf_obj_pin(map_fd, QUEUE_PIN_PATH), "pin map")) + goto out; + + ret = get_bpftool_command_output("map dump pinned " QUEUE_PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "plain dump fails"); + ASSERT_NULL(strstr(output, "Found"), "plain dump prints no count"); + + memset(output, 0, sizeof(output)); + get_bpftool_command_output("map dump pinned " QUEUE_PIN_PATH " 2>&1 >/dev/null", + output, sizeof(output)); + ASSERT_HAS_SUBSTR(output, "can't dump queue map", + "plain dump explains why on stderr"); + +out: + unlink(QUEUE_PIN_PATH); + close(map_fd); +} + +void test_bpftool_map_dump(void) +{ + if (test__start_subtest("sk_storage")) + test_sk_storage(); + if (test__start_subtest("queue")) + test_queue(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c b/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c new file mode 100644 index 0000000..e9724c5 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c
@@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <ctype.h> +#include <fcntl.h> +#include <poll.h> +#include <signal.h> +#include <sys/mman.h> +#include <sys/wait.h> +#include <test_progs.h> +#include "bpftool_helpers.h" +#include "bpftool_ringbuf.skel.h" + +#define WAIT_STEPS 500 +#define WAIT_US 10000 + +struct consumer { + pid_t pid; + int fd; +}; + +static void consumer_cleanup(struct consumer *child) +{ + if (child->pid > 0) { + kill(child->pid, SIGKILL); + while (waitpid(child->pid, NULL, 0) < 0 && errno == EINTR) + ; + child->pid = -1; + } + if (child->fd >= 0) { + close(child->fd); + child->fd = -1; + } +} + +static bool consumer_start(struct consumer *child, int map_fd, const char *format, + const char *pin_path, const char *option, + bool pair, bool capture_errors) +{ + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + char path[PATH_MAX]; + char *argv[12], id[16]; + int out[2], ready[2], n = 0, err, err_fd; + struct pollfd pfd; + + if (!ASSERT_OK(detect_bpftool_path(path, sizeof(path)), "bpftool path") || + !ASSERT_OK(bpf_map_get_info_by_fd(map_fd, &info, &len), "map info")) + return false; + snprintf(id, sizeof(id), "%u", info.id); + argv[n++] = path; + if (format) + argv[n++] = (char *)format; + argv[n++] = "map"; + argv[n++] = "event_pipe"; + argv[n++] = pin_path ? "pinned" : "id"; + argv[n++] = pin_path ? (char *)pin_path : id; + if (option) { + argv[n++] = (char *)option; + argv[n++] = "0"; + if (pair) { + argv[n++] = "index"; + argv[n++] = "0"; + } + } + argv[n] = NULL; + if (!ASSERT_OK(pipe2(out, O_CLOEXEC), "output pipe")) + return false; + if (!ASSERT_OK(pipe2(ready, O_CLOEXEC), "exec pipe")) { + close(out[0]); + close(out[1]); + return false; + } + child->pid = fork(); + if (!child->pid) { + close(out[0]); + close(ready[0]); + err_fd = capture_errors ? out[1] : open("/dev/null", O_WRONLY); + if (dup2(out[1], STDOUT_FILENO) < 0 || + dup2(err_fd, STDERR_FILENO) < 0) + goto exec_fail; + if (!capture_errors) + close(err_fd); + close(out[1]); + execv(path, argv); +exec_fail: + err = errno; + if (write(ready[1], &err, sizeof(err)) != sizeof(err)) + _exit(126); + _exit(127); + } + close(out[1]); + close(ready[1]); + child->fd = out[0]; + pfd = (struct pollfd) { .fd = ready[0], .events = POLLIN }; + /* EOF on the close-on-exec pipe distinguishes exec from inherited handlers. */ + err = child->pid > 0 ? poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000) : -1; + if (!ASSERT_GT(child->pid, 0, "fork") || + !ASSERT_GT(err, 0, "exec timeout") || + !ASSERT_EQ(read(ready[0], &err, sizeof(err)), 0, "exec")) { + close(ready[0]); + consumer_cleanup(child); + return false; + } + close(ready[0]); + return true; +} + +static bool consumer_finish(struct consumer *child, int signo, bool success, + char *output, size_t size) +{ + int status = 0, i; + pid_t ret = 0; + ssize_t n; + size_t used = 0; + + if (!ASSERT_OK(fcntl(child->fd, F_SETFL, O_NONBLOCK), "nonblocking output")) + return false; + if (signo && !ASSERT_OK(kill(child->pid, signo), "signal consumer")) + return false; + for (i = 0; i < WAIT_STEPS; i++) { + while (used < size - 1 && + (n = read(child->fd, output + used, size - 1 - used)) > 0) + used += n; + ret = waitpid(child->pid, &status, WNOHANG); + if (ret == child->pid) + break; + if (ret < 0 && errno != EINTR) + break; + usleep(WAIT_US); + } + if (!ASSERT_EQ(ret, child->pid, "bounded consumer exit")) + return false; + child->pid = -1; + while (used < size - 1 && (n = read(child->fd, output + used, size - 1 - used)) > 0) + used += n; + output[used] = '\0'; + return ASSERT_TRUE(WIFEXITED(status), "normal exit") && + ASSERT_EQ(WEXITSTATUS(status) == 0, success, "exit status"); +} + +static bool consumer_ready(struct consumer *child) +{ + unsigned long long caught; + char path[64], line[256]; + int i; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", child->pid); + for (i = 0; i < WAIT_STEPS; i++) { + f = fopen(path, "r"); + if (!f) + break; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "SigCgt: %llx", &caught) == 1 && + (caught & (1ULL << (SIGINT - 1))) && + (caught & (1ULL << (SIGTERM - 1)))) { + fclose(f); + return true; + } + } + fclose(f); + usleep(WAIT_US); + } + return ASSERT_TRUE(false, "consumer signal handlers ready"); +} + +static bool emit_record(struct bpftool_ringbuf *skel, int record) +{ + char packet[64] = {}; + + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = packet, + .data_size_in = sizeof(packet), + ); + + skel->bss->record = record; + return ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.produce), + &opts), "produce record") && + ASSERT_OK(skel->bss->output_err, "ringbuf output"); +} + +static bool consumed(unsigned long *position, unsigned long expected) +{ + int i; + + for (i = 0; i < WAIT_STEPS; i++) { + if (__atomic_load_n(position, __ATOMIC_ACQUIRE) == expected) + return true; + usleep(WAIT_US); + } + return ASSERT_EQ(*position, expected, "consumer position"); +} + +static void check_json(char *output, const char *expected) +{ + char *src = output, *dst = output; + bool quoted = false, escaped = false; + + /* Ignore formatting whitespace while checking the entire JSON document. */ + while (*src) { + if (quoted || !isspace((unsigned char)*src)) + *dst++ = *src; + if (!escaped && *src == '"') + quoted = !quoted; + escaped = !escaped && quoted && *src == '\\'; + src++; + } + *dst = '\0'; + ASSERT_STREQ(output, expected, "JSON records"); +} + +static void test_consumer(const char *format, bool idle, int signo, bool pinned) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + struct bpftool_ringbuf *skel; + unsigned long *position = MAP_FAILED; + int page_size = getpagesize(), fd; + char pin_dir[] = "/sys/fs/bpf/bpftool_ringbuf_XXXXXX"; + char pin_path[sizeof(pin_dir) + sizeof("/map")]; + bool dir_created = false, map_pinned = false; + char output[4096]; + struct pollfd pfd; + + skel = bpftool_ringbuf__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_map__set_max_entries(skel->maps.ringbuf, page_size); + if (!ASSERT_OK(bpftool_ringbuf__load(skel), "load")) + goto out; + fd = bpf_map__fd(skel->maps.ringbuf); + if (pinned) { + if (!ASSERT_OK_PTR(mkdtemp(pin_dir), "create pin directory")) + goto out; + dir_created = true; + snprintf(pin_path, sizeof(pin_path), "%s/map", pin_dir); + if (!ASSERT_OK(bpf_obj_pin(fd, pin_path), "pin ringbuf")) + goto out; + map_pinned = true; + } + position = mmap(NULL, page_size, PROT_READ, MAP_SHARED, fd, 0); + if (!ASSERT_NEQ(position, MAP_FAILED, "consumer mmap")) + goto out; + if (!idle && (!emit_record(skel, 0) || !emit_record(skel, 1))) + goto out; + if (!consumer_start(&child, fd, format, pinned ? pin_path : NULL, + NULL, false, false) || + !consumer_ready(&child)) + goto out; + if (!idle) { + /* Both prefilled records occupy 16 bytes including their headers. */ + if (!consumed(position, 32)) + goto out; + pfd = (struct pollfd) { .fd = child.fd, .events = POLLIN }; + if (!ASSERT_GT(poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000), 0, + "records flushed before exit") || + !ASSERT_TRUE(pfd.revents & POLLIN, "record output readable") || + !emit_record(skel, 2) || !consumed(position, 64)) + goto out; + } + if (!consumer_finish(&child, signo, true, output, sizeof(output))) + goto out; + if (format && !strcmp(format, "-p")) + ASSERT_STREQ(output, idle ? "[]\n" : + "[{\n" + " \"size\": 2,\n" + " \"data\": [0,255\n" + " ]\n" + " },{\n" + " \"size\": 5,\n" + " \"data\": [1,2,3,4,5\n" + " ]\n" + " },{\n" + " \"size\": 17,\n" + " \"data\": [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32\n" + " ]\n" + " }\n" + "]\n", "pretty JSON records"); + else if (format) + check_json(output, idle ? "[]" : + "[{\"size\":2,\"data\":[0,255]}," + "{\"size\":5,\"data\":[1,2,3,4,5]}," + "{\"size\":17,\"data\":[16,17,18,19,20,21,22,23," + "24,25,26,27,28,29,30,31,32]}]"); + else + ASSERT_STREQ(output, idle ? "" : + "== size: 2 =====\n00 ff\n" + "== size: 5 =====\n01 02 03 04 05\n" + "== size: 17 =====\n10 11 12 13 14 15 16 17 " + "18 19 1a 1b 1c 1d 1e 1f\n20\n", "plain records"); +out: + consumer_cleanup(&child); + if (position != MAP_FAILED) + munmap(position, page_size); + if (map_pinned) + ASSERT_OK(unlink(pin_path), "unpin ringbuf"); + if (dir_created) + ASSERT_OK(rmdir(pin_dir), "remove pin directory"); + bpftool_ringbuf__destroy(skel); +} + +static void test_perf_consumer(void) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + unsigned long long seconds, nanoseconds; + struct bpftool_ringbuf *skel; + char output[16384], expected[16384]; + int nr_cpus, cpu, index, offset = 0, i, used = 0, fields; + struct pollfd pfd; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "possible cpus")) + return; + skel = bpftool_ringbuf__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize()); + bpf_map__set_max_entries(skel->maps.perfbuf, nr_cpus); + if (!ASSERT_OK(bpftool_ringbuf__load(skel), "load") || + !consumer_start(&child, bpf_map__fd(skel->maps.perfbuf), NULL, + NULL, NULL, false, false) || !consumer_ready(&child) || + !emit_record(skel, 3)) + goto out; + /* One large record flushes the existing buffered perf output callback. */ + pfd = (struct pollfd) { .fd = child.fd, .events = POLLIN }; + if (!ASSERT_GT(poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000), 0, + "perf record output") || + !ASSERT_TRUE(pfd.revents & POLLIN, "perf output readable") || + !consumer_finish(&child, SIGINT, true, output, sizeof(output))) + goto out; + fields = sscanf(output, "== @%llu.%llu CPU: %d index: %d =====\n%n", + &seconds, &nanoseconds, &cpu, &index, &offset); + if (!ASSERT_EQ(fields, 4, "perf header") || + !ASSERT_GT(offset, 0, "perf payload offset")) + goto out; + ASSERT_GT(seconds * 1000000000ULL + nanoseconds, 0, "perf timestamp"); + ASSERT_LT(nanoseconds, 1000000000ULL, "perf timestamp nanoseconds"); + ASSERT_GE(cpu, 0, "perf cpu"); + ASSERT_LT(cpu, nr_cpus, "perf cpu range"); + ASSERT_EQ(index, cpu, "perf index"); + for (i = 0; i < sizeof(skel->rodata->perf_payload); i++) { + const char *separator = !i ? "" : !(i % 16) ? "\n" : + !(i % 8) ? " " : " "; + + used += snprintf(expected + used, sizeof(expected) - used, + "%s%02x", separator, i == 1 ? 0xff : 0); + } + snprintf(expected + used, sizeof(expected) - used, "\n"); + ASSERT_STREQ(output + offset, expected, "perf payload"); +out: + consumer_cleanup(&child); + bpftool_ringbuf__destroy(skel); +} + +static void test_reject(enum bpf_map_type type, const char *option, bool pair, bool json) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + bool ring = type == BPF_MAP_TYPE_RINGBUF || type == BPF_MAP_TYPE_USER_RINGBUF; + const char *expected = option && !strcmp(option, "foobar") ? + "{\"error\":\"what is 'foobar'?\"}" : option ? + "{\"error\":\"ring buffer maps do not support cpu or index arguments\"}" : + "{\"error\":\"map is not a perf event array or ring buffer\"}"; + char output[4096]; + int fd; + + fd = bpf_map_create(type, NULL, ring ? 0 : 4, ring ? 0 : 4, + ring ? getpagesize() : 1, NULL); + if (!ASSERT_GE(fd, 0, "create map")) + return; + if (consumer_start(&child, fd, json ? "-j" : NULL, NULL, option, pair, true) && + consumer_finish(&child, 0, false, output, sizeof(output))) { + if (json) + check_json(output, expected); + else + ASSERT_GT(strlen(output), 0, "error diagnostic"); + } + consumer_cleanup(&child); + close(fd); +} + +void test_bpftool_ringbuf(void) +{ + if (test__start_subtest("perf_event_array")) + test_perf_consumer(); + if (test__start_subtest("plain")) + test_consumer(NULL, false, SIGINT, false); + if (test__start_subtest("json")) + test_consumer("-j", false, SIGTERM, false); + if (test__start_subtest("pretty_json")) + test_consumer("-p", false, SIGINT, false); + if (test__start_subtest("pinned_plain")) + test_consumer(NULL, false, SIGTERM, true); + if (test__start_subtest("pinned_json")) + test_consumer("-j", false, SIGINT, true); + if (test__start_subtest("pinned_pretty_json")) + test_consumer("-p", false, SIGTERM, true); + if (test__start_subtest("idle_sigint")) + test_consumer(NULL, true, SIGINT, false); + if (test__start_subtest("idle_sigterm_json")) + test_consumer("-j", true, SIGTERM, false); + if (test__start_subtest("idle_sigint_pretty_json")) + test_consumer("-p", true, SIGINT, false); + if (test__start_subtest("reject_array")) + test_reject(BPF_MAP_TYPE_ARRAY, NULL, false, false); + if (test__start_subtest("reject_user_ringbuf")) + test_reject(BPF_MAP_TYPE_USER_RINGBUF, NULL, false, false); + if (test__start_subtest("reject_cpu")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", false, false); + if (test__start_subtest("reject_index")) + test_reject(BPF_MAP_TYPE_RINGBUF, "index", false, false); + if (test__start_subtest("reject_cpu_index")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", true, false); + if (test__start_subtest("reject_user_ringbuf_json")) + test_reject(BPF_MAP_TYPE_USER_RINGBUF, NULL, false, true); + if (test__start_subtest("reject_unknown_json")) + test_reject(BPF_MAP_TYPE_RINGBUF, "foobar", false, true); + if (test__start_subtest("reject_cpu_index_json")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", true, true); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 67b9015..24ab62b 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c
@@ -424,7 +424,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid type", }, { - .descr = "global data test #8, invalid var size", + .descr = "global data test #8, var is smaller than its type", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -457,8 +457,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", }, { .descr = "global data test #9, invalid var size", @@ -498,7 +496,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid size", }, { - .descr = "global data test #10, invalid var size", + .descr = "global data test #10, section is smaller than map value", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -531,8 +529,7 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", + .map_create_err = true, }, { .descr = "global data test #11, multiple section members", @@ -1987,14 +1984,14 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "typedef (invalid name, invalid identifier)", + .descr = "typedef (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPEDEF_ENC(NAME_TBD, 1), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!int", - .str_sec_size = sizeof("\0__!int"), + .str_sec = "\0__\7int", + .str_sec_size = sizeof("\0__\7int"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "typedef_check_btf", .key_size = sizeof(int), @@ -2112,15 +2109,15 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "fwd type (invalid name, invalid identifier)", + .descr = "fwd type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FWD, 0, 0), 0), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!skb", - .str_sec_size = sizeof("\0__!skb"), + .str_sec = "\0__\7skb", + .str_sec_size = sizeof("\0__\7skb"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "fwd_type_check_btf", .key_size = sizeof(int), @@ -2175,7 +2172,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct type (invalid name, invalid identifier)", + .descr = "struct type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2183,8 +2180,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2217,7 +2214,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct member (invalid name, invalid identifier)", + .descr = "struct member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2225,8 +2222,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A\0B*", - .str_sec_size = sizeof("\0A\0B*"), + .str_sec = "\0A\0B\7", + .str_sec_size = sizeof("\0A\0B\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2260,7 +2257,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum type (invalid name, invalid identifier)", + .descr = "enum type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2269,8 +2266,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2306,7 +2303,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum member (invalid name, invalid identifier)", + .descr = "enum member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(0, @@ -2315,8 +2312,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!", - .str_sec_size = sizeof("\0A!"), + .str_sec = "\0A\7", + .str_sec_size = sizeof("\0A\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2625,14 +2622,14 @@ static struct btf_raw_test raw_tests[] = { .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ - /* void (*)(int a, unsigned int !!!) */ + /* void (*)(int a, unsigned int \7) */ BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), BTF_END_RAW, }, - .str_sec = "\0a\0!!!", - .str_sec_size = sizeof("\0a\0!!!"), + .str_sec = "\0a\0\7", + .str_sec_size = sizeof("\0a\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_proto_type_check_btf", .key_size = sizeof(int), @@ -2775,12 +2772,12 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), - /* void !!!(int a, unsigned int b) */ + /* void \7(int a, unsigned int b) */ BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ BTF_END_RAW, }, - .str_sec = "\0a\0b\0!!!", - .str_sec_size = sizeof("\0a\0b\0!!!"), + .str_sec = "\0a\0b\0\7", + .str_sec_size = sizeof("\0a\0b\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_type_check_btf", .key_size = sizeof(int), @@ -2793,7 +2790,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "func (Some arg has no name)", + .descr = "func (Some arg of global func has no name)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ @@ -2802,7 +2799,8 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(0, 2), /* void func(int a, unsigned int) */ - BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_TYPE_ENC(NAME_TBD, /* [4] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 3), BTF_END_RAW, }, .str_sec = "\0a\0func", @@ -2819,6 +2817,57 @@ static struct btf_raw_test raw_tests[] = { }, { + .descr = "func (Some arg of static func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + /* static void func(int a, unsigned int) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func (vararg of global func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* void (*)(int a, ...) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [2] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 0), + /* void func(int a, ...) */ + BTF_TYPE_ENC(NAME_TBD, /* [3] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ .descr = "func (Non zero vlen)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -3585,15 +3634,28 @@ static struct btf_raw_test raw_tests[] = { .btf_load_err = true, }, { - .descr = "type name '?foo' is not ok", + .descr = "type name '?foo' is ok", .raw_types = { /* union ?foo; */ BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_FWD, 1, 0), 0), /* [1] */ BTF_END_RAW, }, BTF_STR_SEC("\0?foo"), - .err_str = "Invalid name", - .btf_load_err = true, +}, +{ + .descr = "names of Rust types and functions are ok", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), 0, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 1, 0), + BTF_FWD_ENC(NAME_NTH(4), 0), /* [3] */ + BTF_TYPEDEF_ENC(NAME_NTH(5), 2), /* [4] */ + BTF_FUNC_PROTO_ENC(0, 1), /* [5] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(6), 1), + BTF_FUNC_ENC(NAME_NTH(7), 5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0u32\0Option<&str>\0__0\0*const str\0{impl#9}<[u8; 4]>\0self\0fmt<str>"), }, { @@ -7063,6 +7125,132 @@ static struct btf_dedup_test dedup_tests[] = { }, }, { + .descr = "dedup: array element comparison", + .input = { + .raw_types = { + /* signed int */ + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + /* signed int[1] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + /* duplicate signed int[1] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [4] */ + /* unsigned int[1] */ + BTF_TYPE_ARRAY_ENC(2, 2, 1), /* [5] */ + /* struct s { signed int a[1]; signed int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + /* struct s { signed int a[1]; unsigned int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [7] */ + BTF_MEMBER_ENC(NAME_NTH(4), 4, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 5, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(2, 2, 1), /* [4] */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 4, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, +}, +{ + .descr = "dedup: array element comparison with same index type", + .input = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [4] */ + BTF_TYPE_ARRAY_ENC(2, 1, 1), /* [5] */ + /* struct s { int a[1]; int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + /* struct s { int a[1]; unsigned int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [7] */ + BTF_MEMBER_ENC(NAME_NTH(4), 4, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 5, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(2, 1, 1), /* [4] */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 4, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, +}, +{ + .descr = "dedup: identical array element comparison", + .input = { + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* struct container { struct elem first[1]; struct elem second[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 3, 32), + /* struct elem[1] */ + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [3] */ + /* struct elem { int x; } */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [4] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + /* duplicate struct container */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(3), 6, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 7, 32), + /* duplicate struct elem[1] */ + BTF_TYPE_ARRAY_ENC(8, 1, 1), /* [6] */ + BTF_TYPE_ARRAY_ENC(9, 1, 1), /* [7] */ + /* duplicate struct elem */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [8] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [9] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0container\0first\0second\0elem\0x"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 3, 32), + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [3] */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [4] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0container\0first\0second\0elem\0x"), + }, +}, +{ .descr = "dedup: struct example #1", /* * struct s {
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c index 9d61611..5c15a6b 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c
@@ -554,6 +554,120 @@ static void test_split_module(void) btf__free(vmlinux_btf); } +static void test_split_loc(void) +{ + struct btf *btf1, *btf2; + int err; + + btf1 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) + return; + + btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */ + + + btf__add_int(btf1, "long", 8, BTF_INT_SIGNED); /* [1] long */ + btf__add_ptr(btf1, 1); /* [2] ptr to long */ + btf__add_func_proto(btf1, 1); /* [3] long (*)(long, long *); */ + btf__add_func_param(btf1, "p1", 1); + btf__add_func_param(btf1, "p2", 2); + btf__add_func(btf1, "foo", BTF_FUNC_STATIC, 3); /* [4] long foo(long, long *); */ + btf__add_loc_param(btf1, 8, BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf1, 3735928559); + btf__add_loc_param_value(btf1, 4277009102); /* [5] loc value */ + btf__add_loc_param(btf1, 8, BTF_LOC_PARAM_REG); /* [6] loc reg 1 */ + btf__add_loc_param_value(btf1, 1); + btf__add_loc_proto(btf1); /* [7] loc proto */ + btf__add_loc_proto_param(btf1, 5); /* param value */ + btf__add_loc_proto_param(btf1, 6); /* param reg 1 */ + + VALIDATE_RAW_BTF( + btf1, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6"); + + btf2 = btf__new_empty_split(btf1); + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) + goto cleanup; + btf__add_loc_param(btf2, 8, BTF_LOC_PARAM_REG); + btf__add_loc_param_value(btf2, 1); /* [8] loc reg 1 */ + btf__add_loc_proto(btf2); /* [9] loc proto */ + btf__add_loc_proto_param(btf2, 5); /* param value */ + btf__add_loc_proto_param(btf2, 8); /* param reg 1 */ + btf__add_locsec(btf2, "inline.text"); /* [10] locsec "inline.text" */ + /* add duplicate locsec */ + btf__add_locsec_loc(btf2, 4, 9, 128); + btf__add_locsec(btf2, "inline.text"); /* [11] locsec "inline.text" */ + btf__add_locsec_loc(btf2, 4, 9, 128); + + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6", + "[8] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[9] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=8", + "[10] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=9 offset=128", + "[11] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=9 offset=128"); + + err = btf__dedup(btf2, NULL); + if (!ASSERT_OK(err, "btf_dedup")) + goto cleanup; + + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6", + "[8] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=7 offset=128", + "[9] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=7 offset=128"); + +cleanup: + btf__free(btf2); + btf__free(btf1); +} + void test_btf_dedup_split() { if (test__start_subtest("split_simple")) @@ -566,4 +680,6 @@ void test_btf_dedup_split() test_split_dup_struct_in_cu(); if (test__start_subtest("split_module")) test_split_module(); + if (test__start_subtest("split_loc")) + test_split_loc(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_distill.c b/tools/testing/selftests/bpf/prog_tests/btf_distill.c index fb67ae1..f0900b4 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_distill.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_distill.c
@@ -671,6 +671,110 @@ static void test_distilled_base_embedded_err(void) btf__free(btf1); } +/* LOC_PARAM, LOC_PROTO should be added to split BTF. */ +static void test_distilled_loc(void) +{ + struct btf *btf1 = NULL, *btf2 = NULL, *btf3 = NULL; + struct btf *btf4 = NULL, *btf5 = NULL; + + btf1 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) + return; + + btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */ + btf__add_func_proto(btf1, 1); /* [2] int (*)(int); */ + btf__add_func_param(btf1, "p1", 1); + btf__add_func(btf1, "foo", BTF_FUNC_STATIC, 2); /* [3] int foo(int); */ + btf__add_loc_param(btf1, 4, BTF_LOC_PARAM_SIGNED | BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf1, -1); /* [4] loc value */ + btf__add_loc_proto(btf1); /* [5] loc proto */ + btf__add_loc_proto_param(btf1, 4); /* param value */ + btf__add_int(btf1, "unsigned int", 4, 0); /* [6] unsigned int */ + + VALIDATE_RAW_BTF( + btf1, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[3] FUNC 'foo' type_id=2 linkage=static", + "[4] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[5] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=4", + "[6] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)"); + + btf2 = btf__new_empty_split(btf1); + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) + goto cleanup; + + btf__add_locsec(btf2, "inline.text"); /* [6] locsec */ + btf__add_locsec_loc(btf2, 3, 5, 256); /* "foo" offset 256 */ + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[3] FUNC 'foo' type_id=2 linkage=static", + "[4] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[5] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=4", + "[6] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)", + "[7] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=3 loc_proto_type_id=5 offset=256"); + + if (!ASSERT_EQ(0, btf__distill_base(btf2, &btf3, &btf4), + "distilled_base") || + !ASSERT_OK_PTR(btf3, "distilled_base") || + !ASSERT_OK_PTR(btf4, "distilled_split") || + !ASSERT_EQ(2, btf__type_cnt(btf3), "distilled_base_type_cnt")) + goto cleanup; + + VALIDATE_RAW_BTF( + btf4, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + /* remainder is split BTF */ + "[2] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=6 offset=256", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[6] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=5"); + + btf5 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf5, "empty_reloc_btf")) + goto cleanup; + btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [1] int */ + btf__add_int(btf5, "char", 1, 0); /* [2] char */ + + if (!ASSERT_EQ(btf__relocate(btf4, btf5), 0, "relocate_split")) + goto cleanup; + VALIDATE_RAW_BTF( + btf4, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'char' size=1 bits_offset=0 nr_bits=8 encoding=(none)", + /* remainder is split BTF */ + "[3] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=5 loc_proto_type_id=7 offset=256", + "[4] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[5] FUNC 'foo' type_id=4 linkage=static", + "[6] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[7] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=6"); + +cleanup: + btf__free(btf5); + btf__free(btf4); + btf__free(btf3); + btf__free(btf2); + btf__free(btf1); +} + void test_btf_distill(void) { if (test__start_subtest("distilled_base")) @@ -689,4 +793,6 @@ void test_btf_distill(void) test_distilled_base_vmlinux(); if (test__start_subtest("distilled_endianness")) test_distilled_endianness(); + if (test__start_subtest("distilled_loc")) + test_distilled_loc(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 9f1b50e..35f214e 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c
@@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <bpf/btf.h> - -static int duration = 0; +#include "testing_helpers.h" void btf_dump_printf(void *ctx, const char *fmt, va_list args) { @@ -45,12 +44,75 @@ static int btf_dump_all_types(const struct btf *btf, void *ctx) return err; } +/* + * Expected output is embedded in the test case source, between + * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT markers. A region is either + * plain C, where a declaration doubles as its own expectation, or C wrapped in + * a block comment, where the rendered form differs from the source. All + * regions of a file concatenate into one expectation, compared against one + * whole-file dump. + * + * Returns a malloc'd buffer for the caller to free, or NULL on failure. + */ +static char *read_expected_output(const char *path) +{ + size_t out_sz = 0, line_cap = 0; + char *out = NULL, *line = NULL; + bool in_region = false; + FILE *f, *out_file; + + f = fopen(path, "r"); + if (!f) + return NULL; + + out_file = open_memstream(&out, &out_sz); + if (!out_file) { + fclose(f); + return NULL; + } + + while (getline(&line, &line_cap, f) > 0) { + const char *p; + + if (strstr(line, "START-EXPECTED-OUTPUT")) { + in_region = true; + continue; + } + if (strstr(line, "END-EXPECTED-OUTPUT")) + in_region = false; + if (!in_region) + continue; + + p = line + strspn(line, " \t"); + + /* opening or closing line of a commented out region */ + if (!strncmp(p, "/*", 2) || !strncmp(p, "*/", 2)) + continue; + + /* + * Only a '*' directly after the indentation is a comment + * prefix. Without one the line is taken as it is, leading + * whitespace included. + */ + p = *p == '*' ? p + 1 : line; + + fputs(p, out_file); + } + + free(line); + fclose(f); + fclose(out_file); + return out; +} + static int test_btf_dump_case(int n, struct btf_dump_test_case *t) { - char test_file[256], out_file[256], diff_cmd[1024]; + char *dump = NULL, *expected = NULL; struct btf *btf = NULL; - int err = 0, fd = -1; - FILE *f = NULL; + char test_file[256]; + size_t dump_sz = 0; + int err = 0; + FILE *f; snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file); @@ -69,29 +131,21 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) if (!t->known_ptr_sz) { btf__set_pointer_size(btf, 8); } else { - CHECK(btf__pointer_size(btf) != 8, "ptr_sz", "exp %d, got %zu\n", - 8, btf__pointer_size(btf)); + size_t ptr_sz = btf__pointer_size(btf); + + ASSERT_EQ(ptr_sz, (size_t)8, "ptr_sz"); } - snprintf(out_file, sizeof(out_file), "/tmp/%s.output.XXXXXX", t->file); - fd = mkstemp(out_file); - if (!ASSERT_GE(fd, 0, "create_tmp")) { - err = fd; - goto done; - } - f = fdopen(fd, "w"); - if (CHECK(f == NULL, "open_tmp", "failed to open file: %s(%d)\n", - strerror(errno), errno)) { - close(fd); + f = open_memstream(&dump, &dump_sz); + if (!ASSERT_OK_PTR(f, "open_memstream")) { + err = -errno; goto done; } err = btf_dump_all_types(btf, f); fclose(f); - close(fd); - if (CHECK(err, "btf_dump", "failure during C dumping: %d\n", err)) { + if (!ASSERT_OK(err, "btf_dump")) goto done; - } snprintf(test_file, sizeof(test_file), "progs/%s.c", t->file); if (access(test_file, R_OK) == -1) @@ -100,28 +154,18 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) * without preserving the directory structure. */ snprintf(test_file, sizeof(test_file), "%s.c", t->file); - /* - * Diff test output and expected test output, contained between - * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT lines in test case. - * For expected output lines, everything before '*' is stripped out. - * Also lines containing comment start and comment end markers are - * ignored. - */ - snprintf(diff_cmd, sizeof(diff_cmd), - "awk '/START-EXPECTED-OUTPUT/{out=1;next} " - "/END-EXPECTED-OUTPUT/{out=0} " - "/\\/\\*|\\*\\//{next} " /* ignore comment start/end lines */ - "out {sub(/^[ \\t]*\\*/, \"\"); print}' '%s' | diff -u - '%s'", - test_file, out_file); - err = system(diff_cmd); - if (CHECK(err, "diff", - "differing test output, output=%s, err=%d, diff cmd:\n%s\n", - out_file, err, diff_cmd)) - goto done; - remove(out_file); + expected = read_expected_output(test_file); + if (!ASSERT_OK_PTR(expected, "read_expected_output")) { + err = -errno; + goto done; + } + + ASSERT_TEXT_EQ(dump, expected, "compare_text_to_expected"); done: + free(expected); + free(dump); btf__free(btf); return err; } @@ -175,7 +219,7 @@ static void test_ctx__dump_and_compare(struct test_ctx *t, fflush(t->dump_buf_file); t->dump_buf[t->dump_buf_sz] = 0; /* some libc implementations don't do this */ - ASSERT_STREQ(t->dump_buf, expected_output, message); + ASSERT_TEXT_EQ(t->dump_buf, expected_output, message); } static void test_btf_dump_incremental(void) @@ -266,6 +310,8 @@ static void test_btf_dump_incremental(void) ASSERT_OK(err, "field_ok"); err = btf__add_field(btf, "s", 6, 64, 0); ASSERT_OK(err, "field_ok"); + /* attribute on a type added after the first dump: the later walk indexes it */ + ASSERT_EQ(btf__add_decl_attr(btf, "round2", 7, -1), 8, "round2_attr"); test_ctx__dump_and_compare(&t, "struct s___2 {\n" @@ -274,7 +320,270 @@ static void test_btf_dump_incremental(void) " Y___2 = 1,\n" " } y;\n" " struct s s;\n" -"};\n\n" , "c_dump1"); +"} __attribute__((round2));\n\n", "c_dump1"); + + test_ctx__free(&t); +} + +static void test_btf_dump_decl_tags(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * struct s1 { int f; } __attribute__((attr1)) + * __attribute__((btf_decl_tag("plain_tag"))); + * typedef struct { int f; } __attribute__((a1)) __attribute__((a2)) t1; + * union u1 { int f; } __attribute__((uattr)); + * struct p1 { char c; int i; } __attribute__((packed)) __attribute__((pattr)); + * struct outer { struct { int g; } __attribute__((inner)) in; }; + * struct s2 { int f; } __attribute__((on_s2)); + * struct s3 { int f; } __attribute__((on_s3)); + * + * plus attributes on a var and a func, which btf_dump does not declare. + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_int(btf, "char", 1, BTF_INT_CHAR); + ASSERT_EQ(id, 2, "char_id"); + + /* a named record with one attribute of each form */ + id = btf__add_struct(btf, "s1", 4); + ASSERT_EQ(id, 3, "s1_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s1_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "attr1", 3, -1), 4, "s1_attr"); + /* no kflag: rendered as a btf_decl_tag(), not as a bare attribute */ + ASSERT_EQ(btf__add_decl_tag(btf, "plain_tag", 3, -1), 5, "s1_plain_tag"); + + /* + * an anonymous record behind a typedef, with two attributes. This is + * the case that cannot be expressed by a caller appending to the dump: + * the attribute has to go before the declarator. + */ + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 6, "anon_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "anon_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "a1", 6, -1), 7, "anon_attr1"); + ASSERT_EQ(btf__add_decl_attr(btf, "a2", 6, -1), 8, "anon_attr2"); + id = btf__add_typedef(btf, "t1", 6); + ASSERT_EQ(id, 9, "typedef_id"); + + /* unions are records too */ + id = btf__add_union(btf, "u1", 4); + ASSERT_EQ(id, 10, "u1_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "u1_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "uattr", 10, -1), 11, "u1_attr"); + + /* composes with the packed attribute, which is derived from layout */ + id = btf__add_struct(btf, "p1", 5); + ASSERT_EQ(id, 12, "p1_id"); + ASSERT_OK(btf__add_field(btf, "c", 2, 0, 0), "p1_field_c"); + ASSERT_OK(btf__add_field(btf, "i", 1, 8, 0), "p1_field_i"); + ASSERT_EQ(btf__add_decl_attr(btf, "pattr", 12, -1), 13, "p1_attr"); + + /* an anonymous record inlined at a member, the third emission position */ + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 14, "inner_id"); + ASSERT_OK(btf__add_field(btf, "g", 1, 0, 0), "inner_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "inner", 14, -1), 15, "inner_attr"); + id = btf__add_struct(btf, "outer", 4); + ASSERT_EQ(id, 16, "outer_id"); + ASSERT_OK(btf__add_field(btf, "in", 14, 0, 0), "outer_field"); + + /* not rendered: a var has no declaration in the C output */ + id = btf__add_var(btf, "v", BTF_VAR_GLOBAL_ALLOCATED, 3); + ASSERT_EQ(id, 17, "var_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "var_attr", 17, -1), 18, "var_attr"); + /* not rendered: neither does a func */ + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 19, "proto_id"); + id = btf__add_func(btf, "fn", BTF_FUNC_GLOBAL, 19); + ASSERT_EQ(id, 20, "func_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "func_attr", 20, -1), 21, "func_attr"); + + /* + * Tags added in reverse target order: the tag with the lower type ID + * names the record with the higher one, so the index is in emission + * order only once btf_dump_resize() has sorted it. Without the sort + * the lookups land outside their runs and the records lose their + * attributes. + */ + id = btf__add_struct(btf, "s2", 4); + ASSERT_EQ(id, 22, "s2_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s2_field"); + id = btf__add_struct(btf, "s3", 4); + ASSERT_EQ(id, 23, "s3_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s3_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "on_s3", 23, -1), 24, "s3_attr"); + ASSERT_EQ(btf__add_decl_attr(btf, "on_s2", 22, -1), 25, "s2_attr"); + + test_ctx__dump_and_compare(&t, +"struct s1 {\n" +" int f;\n" +"} __attribute__((attr1)) __attribute__((btf_decl_tag(\"plain_tag\")));\n" +"\n" +"typedef struct {\n" +" int f;\n" +"} __attribute__((a1)) __attribute__((a2)) t1;\n" +"\n" +"union u1 {\n" +" int f;\n" +"} __attribute__((uattr));\n" +"\n" +"struct p1 {\n" +" char c;\n" +" int i;\n" +"} __attribute__((packed)) __attribute__((pattr));\n" +"\n" +"struct outer {\n" +" struct {\n" +" int g;\n" +" } __attribute__((inner)) in;\n" +"};\n" +"\n" +"struct s2 {\n" +" int f;\n" +"} __attribute__((on_s2));\n" +"\n" +"struct s3 {\n" +" int f;\n" +"} __attribute__((on_s3));\n" +"\n", "dump_and_compare"); + + test_ctx__free(&t); +} + +/* + * An attribute on a member is emitted at the member declaration, and one on + * the record at the closing brace. Both are keyed by the record in the index, + * so this also checks that they do not leak into each other's position. + */ +static void test_btf_dump_decl_tags_members(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * struct m1 { + * int (*fp)(void) __attribute__((fp_attr)); + * int a; + * int b __attribute__((b1)) __attribute__((b2)); + * int arr[4] __attribute__((arr_attr)); + * } __attribute__((rec)); + * struct bits { int x : 3 __attribute__((x_attr)); int y : 29; }; + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_array(btf, 1, 1, 4); + ASSERT_EQ(id, 2, "array_id"); + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 3, "proto_id"); + id = btf__add_ptr(btf, 3); + ASSERT_EQ(id, 4, "ptr_id"); + + id = btf__add_struct(btf, "m1", 32); + ASSERT_EQ(id, 5, "m1_id"); + ASSERT_OK(btf__add_field(btf, "fp", 4, 0, 0), "m1_fp"); + ASSERT_OK(btf__add_field(btf, "a", 1, 64, 0), "m1_a"); + ASSERT_OK(btf__add_field(btf, "b", 1, 96, 0), "m1_b"); + ASSERT_OK(btf__add_field(btf, "arr", 2, 128, 0), "m1_arr"); + ASSERT_EQ(btf__add_decl_attr(btf, "rec", 5, -1), 6, "m1_rec_attr"); + /* the attribute follows the whole declarator, pointer and array alike */ + ASSERT_EQ(btf__add_decl_attr(btf, "fp_attr", 5, 0), 7, "m1_fp_attr"); + /* two on one member, to pin the ordering */ + ASSERT_EQ(btf__add_decl_attr(btf, "b1", 5, 2), 8, "m1_b1"); + ASSERT_EQ(btf__add_decl_attr(btf, "b2", 5, 2), 9, "m1_b2"); + ASSERT_EQ(btf__add_decl_attr(btf, "arr_attr", 5, 3), 10, "m1_arr_attr"); + + /* a bit-field: the attribute has to follow the width */ + id = btf__add_struct(btf, "bits", 4); + ASSERT_EQ(id, 11, "bits_id"); + ASSERT_OK(btf__add_field(btf, "x", 1, 0, 3), "bits_x"); + ASSERT_OK(btf__add_field(btf, "y", 1, 3, 29), "bits_y"); + ASSERT_EQ(btf__add_decl_attr(btf, "x_attr", 11, 0), 12, "bits_x_attr"); + + test_ctx__dump_and_compare(&t, +"struct m1 {\n" +" int (*fp)(void) __attribute__((fp_attr));\n" +" int a;\n" +" int b __attribute__((b1)) __attribute__((b2));\n" +" int arr[4] __attribute__((arr_attr));\n" +"} __attribute__((rec));\n" +"\n" +"struct bits {\n" +" int x: 3 __attribute__((x_attr));\n" +" int y: 29;\n" +"};\n" +"\n", "dump_and_compare"); + + test_ctx__free(&t); +} + +/* + * A typedef is the third and last kind btf_dump declares. Its attributes go + * after the declarator, so a typedef of an anonymous record can carry three + * groups in one declaration, binding to three different entities. + */ +static void test_btf_dump_decl_tags_typedef(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * typedef int td __attribute__((t1)) + * __attribute__((btf_decl_tag("t2"))); + * typedef struct { + * int f __attribute__((memb_attr)); + * } __attribute__((rec_attr)) both __attribute__((both_attr)); + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_typedef(btf, "td", 1); + ASSERT_EQ(id, 2, "td_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "t1", 2, -1), 3, "td_attr1"); + /* no kflag: rendered as a btf_decl_tag(), not as a bare attribute */ + ASSERT_EQ(btf__add_decl_tag(btf, "t2", 2, -1), 4, "td_attr2"); + + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 5, "anon_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "anon_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "rec_attr", 5, -1), 6, "anon_rec_attr"); + ASSERT_EQ(btf__add_decl_attr(btf, "memb_attr", 5, 0), 7, "anon_memb_attr"); + id = btf__add_typedef(btf, "both", 5); + ASSERT_EQ(id, 8, "both_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "both_attr", 8, -1), 9, "both_attr"); + + test_ctx__dump_and_compare(&t, +"typedef int td __attribute__((t1)) __attribute__((btf_decl_tag(\"t2\")));\n" +"\n" +"typedef struct {\n" +" int f __attribute__((memb_attr));\n" +"} __attribute__((rec_attr)) both __attribute__((both_attr));\n" +"\n", "dump_and_compare"); test_ctx__free(&t); } @@ -295,8 +604,10 @@ static void test_btf_dump_type_tags(void) * struct s { * void __attribute__((btf_type_tag(\"void_tag\"))) *p1; * void __attribute__((void_attr)) *p2; + * void __arena *p3; * }; * + * The arena tag is rendered with the address space it denotes. */ id = btf__add_type_tag(btf, "void_tag", 0); @@ -309,17 +620,25 @@ static void test_btf_dump_type_tags(void) id = btf__add_ptr(btf, id); ASSERT_EQ(id, 4, "void_ptr_id2"); - id = btf__add_struct(btf, "s", 8); - ASSERT_EQ(id, 5, "struct_id"); + id = btf__add_type_tag(btf, "arena", 0); + ASSERT_EQ(id, 5, "arena_tag_id"); + id = btf__add_ptr(btf, id); + ASSERT_EQ(id, 6, "void_ptr_id3"); + + id = btf__add_struct(btf, "s", 24); + ASSERT_EQ(id, 7, "struct_id"); err = btf__add_field(btf, "p1", 2, 0, 0); ASSERT_OK(err, "field_ok1"); - err = btf__add_field(btf, "p2", 4, 0, 0); + err = btf__add_field(btf, "p2", 4, 64, 0); ASSERT_OK(err, "field_ok2"); + err = btf__add_field(btf, "p3", 6, 128, 0); + ASSERT_OK(err, "field_ok3"); test_ctx__dump_and_compare(&t, "struct s {\n" " void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n" " void __attribute__((void_attr)) *p2;\n" +" void __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\"))) *p3;\n" "};\n\n", "dump_and_compare"); test_ctx__free(&t); @@ -1060,6 +1379,15 @@ void test_btf_dump() { if (test__start_subtest("btf_dump: incremental")) test_btf_dump_incremental(); + if (test__start_subtest("btf_dump: decl_tags")) + test_btf_dump_decl_tags(); + + if (test__start_subtest("btf_dump: decl_tags_members")) + test_btf_dump_decl_tags_members(); + + if (test__start_subtest("btf_dump: decl_tags_typedef")) + test_btf_dump_decl_tags_typedef(); + if (test__start_subtest("btf_dump: type_tags")) test_btf_dump_type_tags();
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c b/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c index 32159d3..dcb5429 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c
@@ -31,8 +31,11 @@ struct field_data { { .ids = { 11 }, .strs = { "decltag" } }, { .ids = { 6 }, .strs = { "typetag" } }, { .ids = {}, .strs = { "e64", "eval1", "eval2", "eval3" } }, - { .ids = { 15, 16 }, .strs = { "datasec1" } } - + { .ids = { 15, 16 }, .strs = { "datasec1" } }, + { .ids = {}, .strs = { "" } }, + { .ids = {}, .strs = { "" } }, + { .ids = { 22, 23 }, .strs = { "" } }, + { .ids = { 14, 24 }, .strs = { ".loc" } } }; /* Fabricate BTF with various types and check BTF field iteration finds types, @@ -88,6 +91,19 @@ void test_btf_field_iter(void) btf__add_datasec_var_info(btf, 15, 0, 4); btf__add_datasec_var_info(btf, 16, 4, 8); + btf__add_loc_param(btf, 4, BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED); + /* [22] loc value -1 */ + btf__add_loc_param_value(btf, -1); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG); /* [23] loc reg 1 */ + btf__add_loc_param_value(btf, 1); + + btf__add_loc_proto(btf); /* [24] loc proto */ + btf__add_loc_proto_param(btf, 22); /* param value -1, */ + btf__add_loc_proto_param(btf, 23); /* param reg 1 */ + + btf__add_locsec(btf, ".loc"); /* [25] locsec ".loc" */ + btf__add_locsec_loc(btf, 14, 24, 128); /* "func" */ + VALIDATE_RAW_BTF( btf, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", @@ -123,7 +139,16 @@ void test_btf_field_iter(void) "\t'eval3' val=3000", "[21] DATASEC 'datasec1' size=12 vlen=2\n" "\ttype_id=15 offset=0 size=4\n" - "\ttype_id=16 offset=4 size=8"); + "\ttype_id=16 offset=4 size=8", + "[22] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[23] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[24] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=22\n" + "\ttype_id=23", + "[25] LOCSEC '.loc' vlen=1\n" + "\tfunc_type_id=14 loc_proto_type_id=24 offset=128"); for (id = 1; id < btf__type_cnt(btf); id++) { struct btf_type *t = btf_type_by_id(btf, id);
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_inline.c b/tools/testing/selftests/bpf/prog_tests/btf_inline.c new file mode 100644 index 0000000..e630e5f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_inline.c
@@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ + +#include <test_progs.h> +#include <bpf/btf.h> +#include <bpf/libbpf.h> + +#define BTF_SYSFS_DIR "/sys/kernel/btf" +#define BTF_INLINE_SUFFIX ".inline" + +/* + * For a specific inline site, verify we have the right function, + * loc proto and loc param representation and that the offset is + * reasonable given the caller where it was inlined. + */ +void test_btf_inline(void) +{ + struct btf *inline_btf = NULL, *btf = NULL, *vmlinux_btf = NULL; + const char *inline_caller = "bpf_testmod_test_write"; + const char *inline_func = "testmod_inline_len"; + unsigned long long caller_addr, base_addr; + bool skip = false, found_loc = false; + const struct btf_loc_param *lp; + int locsec_id, func_id, n, i; + const struct btf_type *t; + struct btf_loc *l; + const __u32 *p; + int err = 0; + + if (!env.has_testmod) { + test__skip(); + return; + } + + /* LOCSEC offsets are relative to the ELF section base. */ + base_addr = module_get_base_addr_sec("bpf_testmod", ".text"); + if (!ASSERT_NEQ(base_addr, 0, "base_addr_nonzero")) + return; + + if (!ASSERT_OK(kallsyms_find(inline_caller, &caller_addr), + "find_inline_caller")) + return; + if (!ASSERT_NEQ(caller_addr, 0, "caller_addr_nonzero")) + return; + if (!ASSERT_GE(caller_addr, base_addr, "caller_addr_ge_base_addr")) + return; + caller_addr -= base_addr; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux_btf")) + return; + + btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod", vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "bpf_testmod_btf")) + goto out; + + inline_btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod" BTF_INLINE_SUFFIX, + btf); + err = libbpf_get_error(inline_btf); + /* pahole may not have inline BTF feature support. */ + if (err == -ENOENT) { + skip = true; + goto out; + } + if (!ASSERT_OK(err, "parse_inline_btf")) + goto out; + locsec_id = btf__find_by_name_kind(inline_btf, "inline.text", BTF_KIND_LOCSEC); + if (locsec_id < 0) { + skip = true; + goto out; + } + func_id = btf__find_by_name_kind(inline_btf, inline_func, BTF_KIND_FUNC); + if (!ASSERT_GT(func_id, 0, "inline_caller_func")) + goto out; + t = btf__type_by_id(inline_btf, locsec_id); + n = btf_vlen(t); + for (i = 0, l = btf_locsec_locs(t); i < n; i++, l++) { + if (l->func == func_id) { + found_loc = true; + break; + } + } + if (!ASSERT_TRUE(found_loc, "found_loc")) + goto out; + if (!ASSERT_GT(l->loc_proto, 0, "loc_proto_id")) + goto out; + if (!ASSERT_GT(l->offset, 0, "loc_offset")) + goto out; + t = btf__type_by_id(inline_btf, l->loc_proto); + if (!ASSERT_OK_PTR(t, "loc_proto_ptr")) + goto out; + if (!ASSERT_EQ(btf_vlen(t), 1, "loc_proto_one_param")) + goto out; + p = btf_loc_proto_params(t); + if (!ASSERT_GT(*p, 0, "loc_proto_has_loc_param")) + goto out; + t = btf__type_by_id(inline_btf, *p); + if (!ASSERT_OK_PTR(t, "loc_param_ptr")) + goto out; + lp = btf_loc_param(t); + if (!ASSERT_TRUE(lp->flags & BTF_LOC_PARAM_REG, "param_has_reg")) + goto out; + if (!ASSERT_GT(l->offset, caller_addr, "inline_gt_caller")) + goto out; + /* simple sanity test to roughly ensure inline site still in function */ + if (!ASSERT_LT(l->offset, caller_addr + 512, "inline_in_caller")) + goto out; +out: + btf__free(inline_btf); + btf__free(btf); + btf__free(vmlinux_btf); + if (skip) + test__skip(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c b/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c new file mode 100644 index 0000000..3189133 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c
@@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> +#include "btf_module_allowlist.skel.h" + +static void test_load_with_list(const char **list, size_t count, + int expected_ret) +{ + struct btf_module_allowlist *skel; + int ret; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = list, + .btf_module_allowlist_cnt = count, + ); + + skel = btf_module_allowlist__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open_opts")) + return; + + ret = btf_module_allowlist__load(skel); + ASSERT_EQ(ret, expected_ret, "btf_module_allowlist__load"); + + btf_module_allowlist__destroy(skel); +} + +static void btf_module_allowlist_default(void) +{ + struct btf_module_allowlist *skel; + int ret; + + skel = btf_module_allowlist__open(); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open")) + return; + + ret = btf_module_allowlist__load(skel); + ASSERT_OK(ret, "btf_module_allowlist__load"); + + btf_module_allowlist__destroy(skel); +} + +static void btf_module_allowlist_allow(void) +{ + const char *mod_list[] = { "bpf_testmod" }; + + test_load_with_list(mod_list, 1, 0); +} + +/* + * A non-NULL allowlist with a count of 0 explicitly disables module BTFs. + * The pointer must be non-NULL to distinguish this from an unspecified + * allowlist. + */ +static void btf_module_allowlist_emptylist(void) +{ + const char *mod_list[] = { NULL }; + + test_load_with_list(mod_list, 0, -ESRCH); +} + +/* + * bpf_testmod is not in the allowlist, so its BTF should not be loaded. + */ +static void btf_module_allowlist_skipunlisted(void) +{ + const char *mod_list[] = { "module_nonexist" }; + struct btf_module_allowlist *skel; + int ret; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = mod_list, + .btf_module_allowlist_cnt = 1, + ); + + skel = btf_module_allowlist__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open_opts")) + return; + + ret = bpf_program__set_attach_target(skel->progs.test_btf_module_allowlist, 0, + "bpf_testmod:bpf_testmod_fentry_test1"); + ASSERT_EQ(ret, -ESRCH, "bpf_program__set_attach_target"); + btf_module_allowlist__destroy(skel); +} + +static void test_invalid_input(const char **list, size_t count, + const char *test_name) +{ + struct btf_module_allowlist *skel; + char assert_name[64]; + int err; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = list, + .btf_module_allowlist_cnt = count, + ); + + snprintf(assert_name, sizeof(assert_name), "%s: open_opts", test_name); + skel = btf_module_allowlist__open_opts(&opts); + err = errno; + if (!ASSERT_NULL(skel, assert_name)) { + btf_module_allowlist__destroy(skel); + return; + } + snprintf(assert_name, sizeof(assert_name), "%s: open_opts err", test_name); + ASSERT_EQ(err, EINVAL, assert_name); +} + +static void btf_module_allowlist_invalidinput(void) +{ + const char *names[] = { NULL }; + const char *empty_names[] = { "" }; + const char *duplicate_names[] = { "bpf_testmod", "bpf_testmod" }; + + test_invalid_input(NULL, 1, "null_list"); + test_invalid_input(names, 1, "null_name"); + test_invalid_input(empty_names, 1, "empty_name"); + test_invalid_input(duplicate_names, 2, "duplicate_name"); + test_invalid_input(names, (size_t)SSIZE_MAX + 1, "large_count"); +} + +void test_btf_module_allowlist(void) +{ + struct btf *vmlinux_btf = NULL; + struct btf *module_btf = NULL; + + if (!env.has_testmod) { + printf("%s:SKIP: bpf_testmod is not available\n", __func__); + test__skip(); + return; + } + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) { + printf("%s:SKIP: vmlinux_btf is not available\n", __func__); + test__skip(); + return; + } + + /* Ensure bpf_testmod BTF is available. */ + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + printf("%s:SKIP: bpf_testmod's BTF is not available\n", __func__); + btf__free(vmlinux_btf); + test__skip(); + return; + } + + btf__free(module_btf); + btf__free(vmlinux_btf); + + if (test__start_subtest("btf_module_allowlist_default")) + btf_module_allowlist_default(); + + if (test__start_subtest("btf_module_allowlist_allow")) + btf_module_allowlist_allow(); + + if (test__start_subtest("btf_module_allowlist_skipunlisted")) + btf_module_allowlist_skipunlisted(); + + if (test__start_subtest("btf_module_allowlist_emptylist")) + btf_module_allowlist_emptylist(); + + if (test__start_subtest("btf_module_allowlist_invalidinput")) + btf_module_allowlist_invalidinput(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_rust.c b/tools/testing/selftests/bpf/prog_tests/btf_rust.c new file mode 100644 index 0000000..daf777c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_rust.c
@@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <bpf/btf.h> +#include "bpftool_helpers.h" + +#define FUNC_NAME "write_fmt<scx_simple::BpfStream>" +#define KSYM_NAME "write_fmt_scx_simple__BpfStream_" + +/* The name of a function of Rust is a part of the name of the program in kallsyms */ +static void test_func_name(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int int_id, proto_id, prog_fd = -1, i; + struct bpf_func_info func_info = {}; + struct bpf_prog_info info = {}; + unsigned long long addr; + __u32 len = sizeof(info); + char sym[128], *p = sym; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "i32", 4, BTF_INT_SIGNED); + ASSERT_GT(int_id, 0, "int"); + proto_id = btf__add_func_proto(btf, int_id); + ASSERT_GT(proto_id, 0, "proto"); + ASSERT_OK(btf__add_func_param(btf, "ctx", int_id), "param"); + func_info.type_id = btf__add_func(btf, FUNC_NAME, BTF_FUNC_STATIC, proto_id); + ASSERT_GT(func_info.type_id, 0, "func"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.prog_btf_fd = btf__fd(btf); + opts.func_info = &func_info; + opts.func_info_cnt = 1; + opts.func_info_rec_size = sizeof(func_info); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, + ARRAY_SIZE(insns), &opts); + if (!ASSERT_GE(prog_fd, 0, "prog load")) + goto out; + if (!ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), "prog info")) + goto out; + if (!info.jited_prog_len) { + test__skip(); + goto out; + } + + p += sprintf(p, "bpf_prog_"); + for (i = 0; i < BPF_TAG_SIZE; i++) + p += sprintf(p, "%02x", info.tag[i]); + sprintf(p, "_%s", KSYM_NAME); + ASSERT_OK(kallsyms_find(sym, &addr), sym); +out: + if (prog_fd >= 0) + close(prog_fd); + btf__free(btf); +} + +#define PIN_PATH "/sys/fs/bpf/btf_rust_piece" + +/* + * A piece of a static that LLVM split has the type of the whole static. + * It's the last variable in the section, so its type ends past the map value. + */ +static void test_piece(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int int_id, struct_id, var_id, piece_id, sec_id, map_fd = -1, key = 0; + __u32 value[2] = { 0x11111111, 0x22222222 }; + char line[256] = {}, out[1024] = {}; + struct btf *btf; + FILE *f = NULL; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "u32", 4, 0); + ASSERT_GT(int_id, 0, "int"); + struct_id = btf__add_struct(btf, "Whole", 8); + ASSERT_GT(struct_id, 0, "struct"); + ASSERT_OK(btf__add_field(btf, "a", int_id, 0, 0), "field"); + ASSERT_OK(btf__add_field(btf, "b", int_id, 32, 0), "field"); + var_id = btf__add_var(btf, "CNT", BTF_VAR_STATIC, int_id); + ASSERT_GT(var_id, 0, "var"); + piece_id = btf__add_var(btf, "WHOLE.1", BTF_VAR_STATIC, struct_id); + ASSERT_GT(piece_id, 0, "piece"); + sec_id = btf__add_datasec(btf, ".bss", sizeof(value)); + ASSERT_GT(sec_id, 0, "datasec"); + ASSERT_OK(btf__add_datasec_var_info(btf, var_id, 0, 4), "var info"); + ASSERT_OK(btf__add_datasec_var_info(btf, piece_id, 4, 4), "piece info"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_value_type_id = sec_id; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, ".bss", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "map create")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, value, 0), "map update")) + goto out; + + /* the variable is printed, the piece is not */ + unlink(PIN_PATH); + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin")) + goto out; + f = fopen(PIN_PATH, "r"); + if (!ASSERT_OK_PTR(f, "open")) + goto out; + while (fgets(line, sizeof(line), f) && line[0] == '#') + ; + ASSERT_HAS_SUBSTR(line, "286331153", "var"); + ASSERT_NULL(strstr(line, "572662306"), "piece"); + + /* the same for bpftool */ + if (!ASSERT_OK(get_bpftool_command_output("map dump pinned " PIN_PATH, out, sizeof(out)), + "bpftool")) + goto out; + ASSERT_HAS_SUBSTR(out, "CNT", "var"); + ASSERT_NULL(strstr(out, "WHOLE.1"), "piece"); +out: + if (f) + fclose(f); + unlink(PIN_PATH); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + +/* Serial: programs are not in kallsyms while another test sets bpf_jit_harden */ +void serial_test_btf_rust(void) +{ + if (test__start_subtest("func_name")) + test_func_name(); + if (test__start_subtest("piece")) + test_piece(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c index 3923e64..3fde859 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c
@@ -3,15 +3,21 @@ #include <test_progs.h> #include <bpf/btf.h> +#include <dirent.h> +#include <limits.h> #include <sys/stat.h> #include <sys/mman.h> #include <fcntl.h> #include <unistd.h> +#define BTF_SYSFS_DIR "/sys/kernel/btf" +#define BTF_INLINE_SUFFIX ".inline" + static void test_btf_mmap_sysfs(const char *path, struct btf *base) { struct stat st; __u64 btf_size, end; + size_t mmap_len = 0; void *raw_data = NULL; int fd = -1; long page_size; @@ -26,6 +32,7 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) btf_size = st.st_size; end = (btf_size + page_size - 1) / page_size * page_size; + mmap_len = btf_size; fd = open(path, O_RDONLY); if (!ASSERT_GE(fd, 0, "open_btf")) @@ -39,11 +46,13 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_shared")) goto cleanup; - raw_data = mmap(NULL, end + 1, PROT_READ, MAP_PRIVATE, fd, 0); + mmap_len = end + 1; + raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0); if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_invalid_size")) goto cleanup; - raw_data = mmap(NULL, end, PROT_READ, MAP_PRIVATE, fd, 0); + mmap_len = end; + raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0); if (!ASSERT_OK_PTR(raw_data, "mmap_btf")) goto cleanup; @@ -70,12 +79,84 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) cleanup: btf__free(btf); if (raw_data && raw_data != MAP_FAILED) - munmap(raw_data, btf_size); + munmap(raw_data, mmap_len); if (fd >= 0) close(fd); } +static void test_btf_inline_sysfs_all(void) +{ + struct btf *vmlinux_btf; + struct dirent *dentry; + DIR *dir; + int err = 0; + + dir = opendir(BTF_SYSFS_DIR); + if (!ASSERT_OK_PTR(dir, "open_btf_sysfs")) + return; + + vmlinux_btf = btf__parse(BTF_SYSFS_DIR "/vmlinux", NULL); + if (!ASSERT_OK_PTR(vmlinux_btf, "parse_vmlinux_btf")) { + closedir(dir); + return; + } + + while ((dentry = readdir(dir)) != NULL) { + struct btf *base_btf = NULL, *module_btf = NULL, *inline_btf = NULL; + char btf_path[PATH_MAX], inline_path[PATH_MAX]; + struct stat st; + + /* Skip ".", ".." and "foo.inline" */ + if (strstr(dentry->d_name, ".")) + continue; + + if (strcmp(dentry->d_name, "vmlinux") == 0) + base_btf = vmlinux_btf; + + if (snprintf(btf_path, sizeof(btf_path), "%s/%s", + BTF_SYSFS_DIR, dentry->d_name) >= sizeof(btf_path) || + snprintf(inline_path, sizeof(inline_path), "%s/%s%s", + BTF_SYSFS_DIR, dentry->d_name, BTF_INLINE_SUFFIX) >= + sizeof(inline_path)) { + ASSERT_FAIL("BTF sysfs path is too long\n"); + break; + } + + if (!base_btf) { + module_btf = btf__parse_split(btf_path, vmlinux_btf); + err = libbpf_get_error(module_btf); + if (err) { + /* A module can be unloaded while its sysfs entry is iterated. */ + if (err == -ENOENT) + continue; + ASSERT_OK(err, "parse_module_btf"); + continue; + } + base_btf = module_btf; + } + if (stat(inline_path, &st)) { + err = errno; + if (err != ENOENT) + ASSERT_OK(err, "stat_inline_btf"); + btf__free(module_btf); + continue; + } + if (base_btf == vmlinux_btf) + test_btf_mmap_sysfs(inline_path, base_btf); + inline_btf = btf__parse_split(inline_path, base_btf); + err = libbpf_get_error(inline_btf); + if (!err) + btf__free(inline_btf); + ASSERT_OK(err, "parse_inline_btf"); + btf__free(module_btf); + } + closedir(dir); + + btf__free(vmlinux_btf); +} + void test_btf_sysfs(void) { test_btf_mmap_sysfs("/sys/kernel/btf/vmlinux", NULL); + test_btf_inline_sysfs_all(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c index 5c84723..3bed3a2 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_write.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c
@@ -606,6 +606,38 @@ static void test_btf_add_btf_split() btf__free(base); } +static void test_btf_attrs(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + btf__add_decl_tag(btf, "tag", 2, -1); + btf__add_decl_attr(btf, "tag", 2, -1); + btf__add_decl_attr(btf, "member_attr", 2, 0); + btf__add_type_tag(btf, "tag", 1); + btf__add_type_attr(btf, "tag", 1); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0", + "[3] DECL_TAG 'tag' type_id=2 component_idx=-1", + "[4] DECL_TAG 'tag' type_id=2 component_idx=-1 kflag=1", + "[5] DECL_TAG 'member_attr' type_id=2 component_idx=0 kflag=1", + "[6] TYPE_TAG 'tag' type_id=1", + "[7] TYPE_TAG 'tag' type_id=1 kflag=1"); + + btf__free(btf); +} + void test_btf_write() { if (test__start_subtest("btf_add")) @@ -614,4 +646,6 @@ void test_btf_write() test_btf_add_btf(); if (test__start_subtest("btf_add_btf_split")) test_btf_add_btf_split(); + if (test__start_subtest("btf_attrs")) + test_btf_attrs(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/call_rcu.c b/tools/testing/selftests/bpf/prog_tests/call_rcu.c new file mode 100644 index 0000000..e4d271d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/call_rcu.c
@@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "call_rcu.skel.h" +#include "call_rcu_fail.skel.h" + +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +/* + * Force the grace period, then poll: the callback still has to be invoked + * afterwards, and call_rcu() is lazy on a CONFIG_RCU_LAZY kernel. + */ +static bool wait_for_callbacks(struct call_rcu *skel, int expected) +{ + int i, got = 0; + + kern_sync_rcu(); + for (i = 0; i < 3000; i++) { + got = __atomic_load_n(&skel->bss->callbacks, __ATOMIC_ACQUIRE); + if (got >= expected) + return true; + usleep(10000); + } + return ASSERT_GE(got, expected, "callbacks"); +} + +static void test_call_rcu_run(bool trace) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_program *prog; + struct call_rcu *skel; + struct elem elem; + __u32 key = 1; + int err; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + prog = trace ? skel->progs.arm_trace : skel->progs.arm; + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"); + ASSERT_EQ(skel->bss->busy_err, -EBUSY, "busy_err"); + + if (!wait_for_callbacks(skel, 1)) + goto out; + + ASSERT_EQ(skel->bss->cb_key, key, "cb_key"); + ASSERT_EQ(skel->bss->cb_val, 0xdeadbeef, "cb_val"); + ASSERT_EQ(skel->bss->cb_max_entries, bpf_map__max_entries(skel->maps.arr), "cb_map"); + + err = bpf_map__lookup_elem(skel->maps.arr, &key, sizeof(key), &elem, sizeof(elem), 0); + if (ASSERT_OK(err, "lookup")) + ASSERT_EQ(elem.val, 0, "value_cleared"); + + /* The head is disarmed before the callback runs, so it can be reused. */ + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "test_run_again")) + goto out; + ASSERT_EQ(skel->bss->arm_err, 0, "rearm_err"); + wait_for_callbacks(skel, 2); +out: + call_rcu__destroy(skel); +} + +static void test_call_rcu_chain(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + skel->bss->chain = 1; + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts), "test_run")) + goto out; + + if (!wait_for_callbacks(skel, 1)) + goto out; + ASSERT_EQ(skel->bss->chain_err, 0, "chain_err"); + wait_for_callbacks(skel, 2); +out: + call_rcu__destroy(skel); +} + +/* + * A callback that keeps re-arming must stop once the map loses its last user + * reference, or it pins the program for good. Hold an independent fd on .bss + * so the refusal is still readable after the skeleton is gone. + */ +static void test_call_rcu_teardown(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int i, err, fd = 0, bss_fd = -1; + struct bpf_prog_info pinfo = {}; + struct bpf_map_info minfo = {}; + __u32 len, prog_id, zero = 0; + struct call_rcu *skel; + char *buf = NULL; + size_t off, vsz; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + len = sizeof(pinfo); + if (!ASSERT_OK(bpf_prog_get_info_by_fd(bpf_program__fd(skel->progs.arm), &pinfo, &len), + "prog_info")) + goto out; + prog_id = pinfo.id; + + len = sizeof(minfo); + if (!ASSERT_OK(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &minfo, &len), + "bss_info")) + goto out; + bss_fd = bpf_map_get_fd_by_id(minfo.id); + if (!ASSERT_GE(bss_fd, 0, "bss_fd")) + goto out; + + vsz = bpf_map__value_size(skel->maps.bss); + off = (char *)&skel->bss->chain_err - (char *)skel->bss; + buf = malloc(vsz); + if (!ASSERT_OK_PTR(buf, "buf")) + goto out; + + skel->bss->chain = INT_MAX; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err")) + goto out; + /* The chain has to be running before the map reference goes away. */ + if (!wait_for_callbacks(skel, 2)) + goto out; + + call_rcu__destroy(skel); + skel = NULL; + + for (i = 0; i < 3000; i++) { + fd = bpf_prog_get_fd_by_id(prog_id); + if (fd < 0) + break; + close(fd); + usleep(10000); + } + if (!ASSERT_EQ(fd, -ENOENT, "prog_freed")) + goto out; + + if (ASSERT_OK(bpf_map_lookup_elem(bss_fd, &zero, buf), "bss_lookup")) + ASSERT_EQ(*(int *)(buf + off), -EPERM, "chain_refused"); +out: + free(buf); + if (bss_fd >= 0) + close(bss_fd); + call_rcu__destroy(skel); +} + +static void test_call_rcu_bad_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + struct call_rcu *skel; + int fd; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + opts.btf_fd = bpf_object__btf_fd(skel->obj); + opts.btf_key_type_id = bpf_map__btf_key_type_id(skel->maps.arr); + opts.btf_value_type_id = bpf_map__btf_value_type_id(skel->maps.arr); + + fd = bpf_map_create(BPF_MAP_TYPE_HASH, "rcu_hash", sizeof(__u32), + bpf_map__value_size(skel->maps.arr), 1, &opts); + ASSERT_EQ(fd, -EOPNOTSUPP, "hash_rejected"); + if (fd >= 0) + close(fd); + + call_rcu__destroy(skel); +} + +/* Iterating would hand the program a writable pointer to the head. */ +static void test_call_rcu_iter(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo = {}; + struct bpf_link *link; + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + linfo.map.map_fd = bpf_map__fd(skel->maps.arr); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump, &opts); + if (!ASSERT_ERR_PTR(link, "iter_rejected")) + bpf_link__destroy(link); + else + ASSERT_EQ(libbpf_get_error(link), -EOPNOTSUPP, "iter_errno"); + + call_rcu__destroy(skel); +} + +static void test_call_rcu_inner_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + struct call_rcu *skel; + int fd; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + opts.inner_map_fd = bpf_map__fd(skel->maps.arr); + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "rcu_outer", + sizeof(__u32), sizeof(__u32), 1, &opts); + ASSERT_EQ(fd, -EOPNOTSUPP, "inner_map_rejected"); + if (fd >= 0) + close(fd); + + call_rcu__destroy(skel); +} + +/* Two heads in the same map must be independent. */ +static void test_call_rcu_two_heads(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_both), &opts), + "test_run")) + goto out; + if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err")) + goto out; + if (!wait_for_callbacks(skel, 2)) + goto out; + ASSERT_EQ(skel->bss->cb_keys, 0x3, "both_keys"); +out: + call_rcu__destroy(skel); +} + +void test_call_rcu(void) +{ + if (test__start_subtest("run")) + test_call_rcu_run(false); + if (test__start_subtest("run_tasks_trace")) + test_call_rcu_run(true); + if (test__start_subtest("chain")) + test_call_rcu_chain(); + if (test__start_subtest("two_heads")) + test_call_rcu_two_heads(); + if (test__start_subtest("teardown")) + test_call_rcu_teardown(); + if (test__start_subtest("bad_map")) + test_call_rcu_bad_map(); + if (test__start_subtest("iter")) + test_call_rcu_iter(); + if (test__start_subtest("inner_map")) + test_call_rcu_inner_map(); + RUN_TESTS(call_rcu_fail); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c new file mode 100644 index 0000000..cc63b84 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c
@@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The kernel replaces the offsets of functions in a frozen read-only map with + * their addresses when the program is loaded. The program has to be the only + * user of the map, and no other program can use the map after that. + */ +#include <test_progs.h> +#include <linux/filter.h> +#include <bpf/btf.h> + +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) + +#define CALLEE_INSN 6 +#define DATA 0x1234 + +/* + * main: r2 = &value; r2 = *(u64 *)(r2 + 8); r1 = 10; callx r2; exit + * add1: r0 = r1; r0 += 1; exit + * + * where value is { DATA, offset of add1 in the program }. + */ +static const struct bpf_insn callx_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_2, 8), + BPF_MOV64_IMM(BPF_REG_1, 10), + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), +}; + +/* reads the data of the map */ +static const struct bpf_insn reader_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0), + BPF_EXIT_INSN(), +}; + +/* refers to the map and is rejected: r0 is not set */ +static const struct bpf_insn bad_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_EXIT_INSN(), +}; + +static char log_buf[16 * 1024]; +static struct bpf_func_info func_info[2]; +static int btf_fd; + +static int create_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + __u64 value[2] = { DATA, CALLEE_INSN * sizeof(struct bpf_insn) }; + int fd, zero = 0; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "callx_rodata", sizeof(int), sizeof(value), 1, + &opts); + if (!ASSERT_OK_FD(fd, "map_create")) + return -1; + if (!ASSERT_OK(bpf_map_update_elem(fd, &zero, value, 0), "map_update") || + !ASSERT_OK(bpf_map_freeze(fd), "map_freeze")) { + close(fd); + return -1; + } + return fd; +} + +static int load(const struct bpf_insn *prog_insns, int cnt, int map_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log_buf, + .log_size = sizeof(log_buf), + .log_level = 1, + ); + struct bpf_insn insns[ARRAY_SIZE(callx_insns)]; + + if (prog_insns == callx_insns) { + /* add1() is referred to by the data only and is found through func_info */ + opts.prog_btf_fd = btf_fd; + opts.func_info = func_info; + opts.func_info_cnt = 2; + opts.func_info_rec_size = sizeof(func_info[0]); + } + memcpy(insns, prog_insns, cnt * sizeof(insns[0])); + insns[0].imm = map_fd; + log_buf[0] = 0; + return bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "callx_map", "GPL", insns, cnt, &opts); +} + +#define LOAD(insns, map_fd) load(insns, ARRAY_SIZE(insns), map_fd) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +static void __test_callx_func_ptr_map(void) +{ + int int_id, proto_id, map_fd = -1, prog_fd = -1, reader_fd = -1, fd, zero = 0; + int main_id, add1_id; + struct btf *btf; + __u64 value[2]; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + proto_id = btf__add_func_proto(btf, int_id); + main_id = btf__add_func(btf, "main_prog", BTF_FUNC_GLOBAL, proto_id); + add1_id = btf__add_func(btf, "add1", BTF_FUNC_STATIC, proto_id); + if (!ASSERT_GT(main_id, 0, "btf_add_main") || + !ASSERT_GT(add1_id, 0, "btf_add_add1") || + !ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + func_info[0].insn_off = 0; + func_info[0].type_id = main_id; + func_info[1].insn_off = CALLEE_INSN; + func_info[1].type_id = add1_id; + btf_fd = btf__fd(btf); + + /* + * Another program relies on what the map has: it's verified with + * the data folded into constants. Pointers to functions are not looked + * for in such map. + */ + map_fd = create_map(); + if (map_fd < 0) + goto out; + reader_fd = LOAD(reader_insns, map_fd); + if (!ASSERT_OK_FD(reader_fd, "load_reader")) + goto out; + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_shared")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "unreachable insn 6", "log_shared"); + run(reader_fd, DATA, "run_reader"); + close(reader_fd); + reader_fd = -1; + close(map_fd); + + map_fd = create_map(); + if (map_fd < 0) + goto out; + + /* a program that is rejected is not a user, libbpf loads it again to get the log */ + fd = LOAD(bad_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_bad")) + close(fd); + + prog_fd = LOAD(callx_insns, map_fd); + if (!ASSERT_OK_FD(prog_fd, "load_callx")) { + printf("%s\n", log_buf); + goto out; + } + run(prog_fd, 11, "run_callx"); + + /* the offset of the function is gone from the map, the data is intact */ + if (ASSERT_OK(bpf_map_lookup_elem(map_fd, &zero, value), "map_lookup")) { + ASSERT_EQ(value[0], DATA, "data"); + ASSERT_NEQ(value[1], CALLEE_INSN * sizeof(struct bpf_insn), "pointer"); + } + + /* no other program can use the map now, another instance of the same one too */ + fd = LOAD(reader_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_reader_after")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "has addresses of functions of another program", "log_reader"); + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_second_instance")) + close(fd); + + run(prog_fd, 11, "run_callx_again"); +out: + if (reader_fd >= 0) + close(reader_fd); + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + +#else + +static void __test_callx_func_ptr_map(void) +{ + test__skip(); +} + +#endif + +void test_callx_func_ptr_map(void) +{ + __test_callx_func_ptr_map(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c new file mode 100644 index 0000000..f6ef67c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c
@@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +#include "callx_rodata.lskel.h" + +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +/* + * Every program that has callx gets its own copy of .rodata with the offsets + * of its functions. The copies have what user space puts into .rodata before + * the load. The program that doesn't have callx uses .rodata. + */ +static void __test_callx_rodata_lskel(void) +{ + struct callx_rodata_lskel *skel; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + skel = callx_rodata_lskel__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + /* gcc doesn't support indirect calls */ + if (skel->rodata->skip) { + test__skip(); + goto out; + } + + skel->rodata->bias = 7; + + if (!ASSERT_OK(callx_rodata_lskel__load(skel), "load")) + goto out; + + skel->bss->op_idx = 0; + run(skel->progs.select_op.prog_fd, 17, "add_bias"); + skel->bss->op_idx = 1; + run(skel->progs.select_op.prog_fd, 30, "mul3"); + skel->bss->op_idx = 2; + run(skel->progs.select_op.prog_fd, -1, "out_of_range"); + /* mul3(add_bias(4)) */ + run(skel->progs.both_ops.prog_fd, 33, "both_ops"); + run(skel->progs.read_bias.prog_fd, 7, "read_bias"); +out: + callx_rodata_lskel__destroy(skel); +} + +#else + +static void __test_callx_rodata_lskel(void) +{ + test__skip(); +} + +#endif + +void test_callx_rodata_lskel(void) +{ + __test_callx_rodata_lskel(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c deleted file mode 100644 index 490e15e..0000000 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ /dev/null
@@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "bpf/libbpf.h" -#include <test_progs.h> -#include <network_helpers.h> - -#include "cb_refs.skel.h" - -static char log_buf[1024 * 1024]; - -struct { - const char *prog_name; - const char *err_msg; -} cb_refs_tests[] = { - { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, - { "leak_prog", "Possibly NULL pointer passed to helper R2" }, - { "nested_cb", "Unreleased reference id=5 alloc_insn=2" }, /* alloc_insn=2{4,5} */ - { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ -}; - -void test_cb_refs(void) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf, - .kernel_log_size = sizeof(log_buf), - .kernel_log_level = 1); - struct bpf_program *prog; - struct cb_refs *skel; - int i; - - for (i = 0; i < ARRAY_SIZE(cb_refs_tests); i++) { - LIBBPF_OPTS(bpf_test_run_opts, run_opts, - .data_in = &pkt_v4, - .data_size_in = sizeof(pkt_v4), - .repeat = 1, - ); - skel = cb_refs__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "cb_refs__open_and_load")) - return; - prog = bpf_object__find_program_by_name(skel->obj, cb_refs_tests[i].prog_name); - bpf_program__set_autoload(prog, true); - if (!ASSERT_ERR(cb_refs__load(skel), "cb_refs__load")) - bpf_prog_test_run_opts(bpf_program__fd(prog), &run_opts); - if (!ASSERT_OK_PTR(strstr(log_buf, cb_refs_tests[i].err_msg), "expected error message")) { - fprintf(stderr, "Expected: %s\n", cb_refs_tests[i].err_msg); - fprintf(stderr, "Verifier: %s\n", log_buf); - } - cb_refs__destroy(skel); - } -}
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c index 2bb5773..ea34285 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c
@@ -465,7 +465,7 @@ struct exposed_hook { .expected_err = EXPECTED_ERR, \ }, -#include "cgroup_getset_retval_hooks.h" +#include "progs/cgroup_getset_retval_hooks.h" #undef BPF_RETVAL_HOOK };
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c index 3bd27d2..cc6ab2c 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c
@@ -110,7 +110,7 @@ static void cleanup_bpffs(void) ASSERT_OK(rmdir(BPFFS_ATTACH_COUNTERS), "rmdir "BPFFS_ATTACH_COUNTERS); /* Unmount bpffs, if it wasn't already mounted when we started */ - if (mounted_bpffs) + if (!mounted_bpffs) return; ASSERT_OK(umount(BPFFS_ROOT), "unmount bpffs"); @@ -151,7 +151,7 @@ static void cleanup_cgroups(void) /* Sets up cgroup hiearchary, returns 0 on success. */ static int setup_hierarchy(void) { - return setup_bpffs() || setup_cgroups(); + return setup_cgroups() || setup_bpffs(); } static void destroy_hierarchy(void)
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c b/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c index bb60704..9ec5e4f 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c
@@ -273,11 +273,64 @@ static void test_link_attach_detach(int atype) close(cg); } +static int find_prog_attach_flags(const struct bpf_prog_query_opts *opts, + __u32 prog_id, __u32 *flags) +{ + __u32 i; + + for (i = 0; i < opts->count; i++) { + if (opts->prog_ids[i] == prog_id) { + *flags = opts->prog_attach_flags[i]; + return 0; + } + } + return -ENOENT; +} + +static void assert_preorder_query_flags(int cg, int atype, + __u32 id1, __u32 id2, + __u32 id3, __u32 id4) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 prog_attach_flags[10] = {}; + __u32 prog_ids[10] = {}; + __u32 flags; + int err; + + optq.prog_ids = prog_ids; + optq.prog_attach_flags = prog_attach_flags; + optq.count = 10; + err = bpf_prog_query_opts(cg, atype, &optq); + if (!ASSERT_OK(err, "prog_query")) + return; + + ASSERT_EQ(optq.count, 4, "count"); + + /* Match by prog_id to avoid relying on query order. */ + err = find_prog_attach_flags(&optq, id1, &flags); + if (ASSERT_OK(err, "find id1")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI, "flags id1"); + + err = find_prog_attach_flags(&optq, id2, &flags); + if (ASSERT_OK(err, "find id2")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI | BPF_F_PREORDER, + "flags id2"); + + err = find_prog_attach_flags(&optq, id3, &flags); + if (ASSERT_OK(err, "find id3")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI | BPF_F_PREORDER, + "flags id3"); + + err = find_prog_attach_flags(&optq, id4, &flags); + if (ASSERT_OK(err, "find id4")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI, "flags id4"); +} + static void test_preorder_prog_attach_detach(int atype) { LIBBPF_OPTS(bpf_prog_attach_opts, opta); LIBBPF_OPTS(bpf_prog_detach_opts, optd); - __u32 fd1, fd2, fd3, fd4; + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; struct cgroup_mprog *skel; int cg, err; @@ -294,6 +347,11 @@ static void test_preorder_prog_attach_detach(int atype) fd3 = bpf_program__fd(skel->progs.getsockopt_3); fd4 = bpf_program__fd(skel->progs.getsockopt_4); + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + assert_mprog_count(cg, atype, 0); LIBBPF_OPTS_RESET(opta, @@ -357,6 +415,8 @@ static void test_preorder_prog_attach_detach(int atype) assert_mprog_count(cg, atype, 4); + assert_preorder_query_flags(cg, atype, id1, id2, id3, id4); + err = bpf_prog_detach_opts(fd4, cg, atype, &optd); ASSERT_OK(err, "prog_detach"); assert_mprog_count(cg, atype, 3); @@ -386,7 +446,7 @@ static void test_preorder_link_attach_detach(int atype) LIBBPF_OPTS(bpf_cgroup_opts, opta); struct bpf_link *link1, *link2, *link3, *link4; struct cgroup_mprog *skel; - __u32 fd2; + __u32 fd2, id1, id2, id3, id4; int cg; cg = test__join_cgroup("/preorder_link_attach_detach"); @@ -399,6 +459,11 @@ static void test_preorder_link_attach_detach(int atype) fd2 = bpf_program__fd(skel->progs.getsockopt_2); + id1 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_1)); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_3)); + id4 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_4)); + assert_mprog_count(cg, atype, 0); LIBBPF_OPTS_RESET(opta, @@ -460,6 +525,8 @@ static void test_preorder_link_attach_detach(int atype) assert_mprog_count(cg, atype, 4); + assert_preorder_query_flags(cg, atype, id1, id2, id3, id4); + bpf_link__destroy(link4); assert_mprog_count(cg, atype, 3);
diff --git a/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c new file mode 100644 index 0000000..fa6ccba --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c
@@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "kfunc_call_test.skel.h" + +/* + * bpf_program__clone() loads a program from a prepared object with the + * fd_array of the object, where module BTF fds start at slot 1 and slot 0 is + * empty. It used to pass the fill index of that array as fd_array_cnt, which + * made the kernel resolve slot 0 and fail the load. + */ +void test_clone_module_kfunc(void) +{ + struct kfunc_call_test *skel; + int err, fd; + + skel = kfunc_call_test__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + + fd = bpf_program__clone(skel->progs.kfunc_call_test1, NULL); + if (ASSERT_GE(fd, 0, "clone")) + close(fd); +out: + kfunc_call_test__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c new file mode 100644 index 0000000..3103706 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c
@@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <errno.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <test_progs.h> + +#include "copy_from_user_bprm.skel.h" + +void test_copy_from_user_bprm(void) +{ + char data[] = "first\0second-argument\0SOME_ENV=a\0OTHER_ENV=something"; + char arg0[] = "first"; + char arg1[] = "second-argument"; + char env0[] = "SOME_ENV=a"; + char env1[] = "OTHER_ENV=something"; + struct copy_from_user_bprm *skel; + pid_t child; + int status; + + skel = copy_from_user_bprm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (!ASSERT_OK(copy_from_user_bprm__attach(skel), "attach")) + goto out; + + child = fork(); + if (!ASSERT_GE(child, 0, "fork")) + goto out; + + if (!child) { + char *const argv[] = { arg0, arg1, NULL }; + char *const envp[] = { env0, env1, NULL }; + + skel->bss->monitored_pid = getpid(); + execvpe("true", argv, envp); + _exit(errno); + } + + if (!ASSERT_EQ(waitpid(child, &status, 0), child, "waitpid")) + goto out; + + if (ASSERT_TRUE(WIFEXITED(status), "child_exited")) + ASSERT_EQ(WEXITSTATUS(status), EPERM, "exec_errno"); + + ASSERT_EQ(skel->bss->bprm_argc, 2, "bprm_argc"); + ASSERT_EQ(skel->bss->bprm_envc, 2, "bprm_envc"); + ASSERT_EQ(skel->bss->data_len, sizeof(data), "data_len"); + ASSERT_EQ(skel->bss->invalid_flags_ret, -EINVAL, "invalid_flags_ret"); + ASSERT_EQ(skel->bss->copy_ret, 0, "copy_ret"); + ASSERT_EQ(skel->bss->arg0_ret, sizeof(arg0), "arg0_ret"); + ASSERT_EQ(skel->bss->arg1_ret, sizeof(arg1), "arg1_ret"); + ASSERT_EQ(skel->bss->env0_ret, sizeof(env0), "env0_ret"); + ASSERT_EQ(skel->bss->env1_ret, sizeof(env1), "env1_ret"); + ASSERT_EQ(memcmp(skel->bss->data, data, sizeof(data)), 0, "data"); + ASSERT_EQ(memcmp(skel->bss->arg0, arg0, sizeof(arg0)), 0, "arg0"); + ASSERT_EQ(memcmp(skel->bss->arg1, arg1, sizeof(arg1)), 0, "arg1"); + ASSERT_EQ(memcmp(skel->bss->env0, env0, sizeof(env0)), 0, "env0"); + ASSERT_EQ(memcmp(skel->bss->env1, env1, sizeof(env1)), 0, "env1"); + +out: + copy_from_user_bprm__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 08963c8..80432c4 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c
@@ -7,6 +7,7 @@ #include <sys/mman.h> #include <sys/syscall.h> #include <bpf/btf.h> +#include "bpftool_helpers.h" static int duration = 0; @@ -985,16 +986,15 @@ static size_t roundup_page(size_t sz) static int run_btfgen(const char *src_btf, const char *dst_btf, const char *objpath) { - char command[4096]; + char args[4096]; int n; - n = snprintf(command, sizeof(command), - "./bpftool gen min_core_btf %s %s %s", + n = snprintf(args, sizeof(args), "gen min_core_btf %s %s %s", src_btf, dst_btf, objpath); - if (n < 0 || n >= sizeof(command)) + if (n < 0 || n >= sizeof(args)) return -1; - return system(command); + return run_bpftool_command(args); } static void run_core_reloc_tests(bool use_btfgen)
diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c new file mode 100644 index 0000000..cb10235 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c
@@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "data_in_arena.skel.h" +#include "data_in_arena_decl.skel.h" +#include "data_in_arena_fail.skel.h" + +static int run_prog(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) + return -1; + return topts.retval; +} + +static void run(struct data_in_arena *skel, int counter) +{ + int i; + + ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); + ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); + ASSERT_EQ(skel->data->counter, counter + 1, "counter"); + ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); + for (i = 0; i < 4; i++) + ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); +} + +static void test_in_arena(void) +{ + struct bpf_map_info info = {}; + struct data_in_arena *skel; + __u32 len = sizeof(info); + struct bpf_map *arena; + size_t sz; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); + ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); + ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); + ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); + ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); + ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); + ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); + + /* initial values are set the usual way */ + skel->data->counter = 100; + + if (!ASSERT_OK(data_in_arena__load(skel), "load")) + goto out; + /* there are no maps behind the sections */ + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), + "rodata map"); + run(skel, 100); + + /* pointers to data next to pointers to functions */ + ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); + ASSERT_EQ(skel->data->counter, 102, "counter"); + + /* alignment of sections and pointers to data in data */ + ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); + ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); + ASSERT_EQ(skel->data->x, 43, "x"); + ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); + ASSERT_EQ(skel->data->px, &skel->data->x, "px"); + + /* format strings of bpf_printk() and BPF_SNPRINTF() */ + ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); + ASSERT_STREQ(skel->bss->out, "43-7", "out"); +out: + data_in_arena__destroy(skel); +} + +/* The object has an arena map and __arena variables */ +static void test_declared_arena(void) +{ + struct data_in_arena_decl *skel; + struct bpf_map *map; + int arenas = 0; + + skel = data_in_arena_decl__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_object__for_each_map(map, skel->obj) + arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; + ASSERT_EQ(arenas, 1, "no second arena"); + skel->data->counter = 6; + if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) + goto out; + ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); + ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); + ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); + ASSERT_EQ(skel->data->counter, 8, "counter"); +out: + data_in_arena_decl__destroy(skel); +} + +/* A pointer in data that can't be made an address of arena fails the load */ +static void test_ptr_to_map(void) +{ + struct data_in_arena_fail *skel; + + skel = data_in_arena_fail__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + ASSERT_ERR(data_in_arena_fail__load(skel), "load"); + data_in_arena_fail__destroy(skel); +} + +/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ +static void test_ptr_to_extern(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* __arena variables and no arena map. There is no skeleton: the open fails. */ +static void test_arena_var_no_map(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ +static void test_not_my_arena(bool pin) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); + struct data_in_arena *skel; + struct bpf_map *arena; + int fd = -1; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + if (pin) { + ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); + } else { + fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); + if (!ASSERT_GE(fd, 0, "map_create")) + goto out; + ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); + } + ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); +out: + if (fd >= 0) + close(fd); + data_in_arena__destroy(skel); +} + +/* The object is there if rustc and clang can build it, see Makefile.buildvars */ +static void test_rust(void) +{ + const char *file = "./data_in_arena_rust.bpf.o"; + struct bpf_object *obj; + + if (access(file, R_OK)) { + test__skip(); + return; + } + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open")) + return; + if (!ASSERT_OK(bpf_object__load(obj), "load")) + goto out; + /* libbpf goes on without BTF when the kernel doesn't take it */ + ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); + ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), + 100 + 20 + 3, "retval"); +out: + bpf_object__close(obj); +} + +void test_data_in_arena(void) +{ +#if !defined(__x86_64__) && !defined(__aarch64__) + /* other JITs don't take BPF_F_ARENA_SCALAR */ + test__skip(); + return; +#endif + if (test__start_subtest("arena")) + test_in_arena(); + if (test__start_subtest("declared_arena")) + test_declared_arena(); + if (test__start_subtest("ptr_to_map")) + test_ptr_to_map(); + if (test__start_subtest("ptr_to_extern")) + test_ptr_to_extern(); + if (test__start_subtest("arena_var_no_map")) + test_arena_var_no_map(); + if (test__start_subtest("pinned_arena")) + test_not_my_arena(true); + if (test__start_subtest("reused_arena")) + test_not_my_arena(false); + if (test__start_subtest("rust")) + test_rust(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c index fb2cea7..1a5dc1a 100644 --- a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c
@@ -188,8 +188,9 @@ static void subtest_dmabuf_iter_check_default_iter(struct dmabuf_iter *skel) ASSERT_EQ(sscanf(line, "%ld", &bufinfo.size), 1, "read size"); } else if (f % FIELD_COUNT == NAME) { - ASSERT_EQ(sscanf(line, "%s", bufinfo.name), 1, - "read name"); + /* Keep an empty name for a buffer without one */ + if (sscanf(line, "%s", bufinfo.name) != 1) + bufinfo.name[0] = '\0'; } else if (f % FIELD_COUNT == EXPORTER) { ASSERT_EQ(sscanf(line, "%31s", bufinfo.exporter), 1, "read exporter");
diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 639866c..1bd15c4 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c
@@ -6,6 +6,7 @@ #include "exceptions_ext.skel.h" #include "exceptions_fail.skel.h" #include "exceptions_assert.skel.h" +#include "exceptions_dead_subprog.skel.h" static char log_buf[1024 * 1024]; @@ -321,7 +322,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_throw_subprog), "subprog"), "set_attach_target")) goto done; - }), "subprog() is not a global function", 0); + }), "Cannot replace static functions", 0); if (test__start_subtest("non-throwing extension -> throwing subprog")) /* non-throwing extension -> throwing subprog : BAD (!global) */ @@ -332,7 +333,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_throw_subprog), "throwing_subprog"), "set_attach_target")) goto done; - }), "throwing_subprog() is not a global function", 0); + }), "Cannot replace static functions", 0); if (test__start_subtest("non-throwing extension -> non-throwing subprog")) /* non-throwing extension -> non-throwing global subprog : OK */ @@ -410,9 +411,15 @@ static void test_exceptions_assertions(void) RUN_TESTS(exceptions_assert); } +static void test_exceptions_dead_subprog(void) +{ + RUN_TESTS(exceptions_dead_subprog); +} + void test_exceptions(void) { test_exceptions_success(); test_exceptions_failure(); test_exceptions_assertions(); + test_exceptions_dead_subprog(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 2523c07..d2f2dcc 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
@@ -164,8 +164,10 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, goto close_prog; close_prog: - for (i = 0; i < prog_cnt; i++) - bpf_link__destroy(link[i]); + if (link) { + for (i = 0; i < prog_cnt; i++) + bpf_link__destroy(link[i]); + } bpf_object__close(obj); bpf_object__close(tgt_obj); free(link); @@ -441,6 +443,20 @@ static void test_func_replace_int_with_void(void) " doesn't match type INT of global_func2()"); } +static void test_func_replace_ret_pair(void) +{ + const char *msg = "Return type of new_agg_ret_target_func() has size 8 " + "and of agg_ret_target_func() size 16"; + + /* + * An extension cannot replace a function whose return value comes back + * in the R0:R2 pair: only R0 is checked at the extension's exit, so it + * would leave R2 stale for the target's callers. + */ + test_obj_load_failure_common("freplace_ret_pair.bpf.o", + "./aggregate_ret_target.bpf.o", msg); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -660,6 +676,8 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_progmap(); if (test__start_subtest("freplace_int_with_void")) test_func_replace_int_with_void(); + if (test__start_subtest("freplace_ret_pair")) + test_func_replace_ret_pair(); if (test__start_subtest("freplace_void")) test_func_replace_void(); if (test__start_subtest("sleepable_fentry_to_xdp"))
diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c index 48aae7e..e59c6c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/file_reader.c +++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c
@@ -7,10 +7,12 @@ #include "file_reader_fail.skel.h" #include <dlfcn.h> #include <sys/mman.h> +#include <sys/stat.h> const char *user_ptr = "hello world"; char file_contents[256000]; void *addr; +__u64 beyond_eof_offset; void *get_executable_base_addr(void) { @@ -26,12 +28,18 @@ void *get_executable_base_addr(void) static int initialize_file_contents(void) { + struct stat st; int fd, page_sz = sysconf(_SC_PAGESIZE); ssize_t n = 0, cur; fd = open("/proc/self/exe", O_RDONLY); if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n")) return 1; + if (!ASSERT_OK(fstat(fd, &st), "fstat /proc/self/exe")) { + close(fd); + return 1; + } + beyond_eof_offset = st.st_size + (1ULL << 30); do { cur = read(fd, file_contents + n, sizeof(file_contents) - n); @@ -75,6 +83,7 @@ static void run_test(const char *prog_name) memcpy(skel->bss->user_buf, file_contents, sizeof(file_contents)); skel->bss->pid = getpid(); + skel->bss->beyond_eof_offset = beyond_eof_offset; err = file_reader__load(skel); if (!ASSERT_OK(err, "file_reader__load")) @@ -110,6 +119,12 @@ void test_file_reader(void) if (test__start_subtest("on_open_validate_file_read")) run_test("on_open_validate_file_read"); + if (test__start_subtest("on_open_non_sleepable_first")) + run_test("on_open_non_sleepable_first"); + + if (test__start_subtest("on_open_sleepable_first")) + run_test("on_open_sleepable_first"); + if (test__start_subtest("negative")) RUN_TESTS(file_reader_fail); }
diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c index fb48926..f6fefb14 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c
@@ -706,7 +706,7 @@ void test_flow_dissector_skb_less_direct_attach(void) struct bpf_flow *skel; struct netns_obj *ns; - ns = netns_new("flow_dissector_skb_less_indirect_attach_ns", true); + ns = netns_new("flow_dissector_skb_less_direct_attach_ns", true); if (!ASSERT_OK_PTR(ns, "create and open netns")) return;
diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c index 80b153d..421dfa6 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c
@@ -27,7 +27,7 @@ #define TEST_NAME_MAX_LEN (32 + SUBTEST_NAME_MAX_LEN) #define MAX_SOURCE_PORTS 3 #define TEST_PACKETS_COUNT 10 -#define TEST_PACKET_LEN 100 +#define TEST_PACKET_LEN 99 #define TEST_PACKET_PATTERN 'a' #define TEST_IPV4 "192.168.0.1/32" #define TEST_IPV6 "100::a/128"
diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 5671c31..7c539cb 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c
@@ -68,12 +68,9 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f { struct test_global_percpu_data__percpu *data = NULL; int i, err, key = 0, num_online, run = 0; - __u64 args[2] = {0x1234ULL, 0x5678ULL}; size_t data_sz; bool *online; LIBBPF_OPTS(bpf_test_run_opts, topts, - .ctx_in = args, - .ctx_size_in = sizeof(args), .flags = BPF_F_TEST_RUN_ON_CPU, ); @@ -81,7 +78,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f if (!ASSERT_OK(err, "parse_cpu_mask_file")) return; - data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data_sz = sizeof(*data); data = calloc(1, data_sz); if (!ASSERT_OK_PTR(data, "calloc percpu data")) goto out; @@ -109,7 +106,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f break; ASSERT_EQ(*runp, ++run, "run"); - ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->cpu_id, i, "cpu_id"); ASSERT_EQ(data->data, 1, "data"); ASSERT_TRUE(data->set, "set"); ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); @@ -127,7 +124,6 @@ static void test_global_percpu_data_init(void) { struct test_global_percpu_data__percpu init_value = {}; struct test_global_percpu_data__percpu *init_data; - const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); struct test_global_percpu_data *skel = NULL; size_t init_data_sz; struct bpf_map *map; @@ -163,9 +159,11 @@ static void test_global_percpu_data_init(void) if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) goto out; + init_value.set = 1; init_value.data = 2; init_value.nums[6] = -1; init_value.struct_data.i = 2; + init_value.struct_data.set = 1; init_value.struct_data.nums[6] = -1; err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); if (!ASSERT_OK(err, "bpf_map__set_initial_value")) @@ -177,35 +175,24 @@ static void test_global_percpu_data_init(void) ASSERT_EQ(init_data->data, init_value.data, "init_value data"); ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->nums[6], init_value.nums[6], "init_value nums[6]"); ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.set, init_value.struct_data.set, + "init_value struct_data.set"); ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], "init_value struct_data.nums[6]"); ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], init_value.nums[6], "skel->percpu->nums[6]"); ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.set, init_value.struct_data.set, + "skel->percpu->struct_data.set"); ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], "skel->percpu->struct_data.nums[6]"); - ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); - err = bpf_map__set_value_size(map, desired_sz); - if (!ASSERT_OK(err, "bpf_map__set_value_size")) - goto out; - if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) - goto out; - if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) - goto out; - - init_data = bpf_map__initial_value(map, &init_data_sz); - if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) - goto out; - if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) - goto out; - if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) - goto out; - err = test_global_percpu_data__load(skel); if (!ASSERT_OK(err, "test_global_percpu_data__load")) goto out; @@ -236,6 +223,67 @@ static void test_global_percpu_data_lskel(void) test_global_percpu_data_lskel__destroy(lskel); } +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +static void test_global_percpu_data_map_resize(void) +{ + const __u32 page_size = sysconf(_SC_PAGE_SIZE); + const __u32 pcpu_max_value_size = ALIGN(PCPU_MIN_UNIT_SIZE, page_size); + const __u32 desired_sz = page_size * 2; + struct test_global_percpu_data *skel; + const int arr_value = -1; + struct bpf_map *map; + int err, prog_fd; + size_t data_sz; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + map = skel->maps.percpu_arr; + skel->percpu_arr->arr[0] = arr_value; + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "bpf_map__value_size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "bpf_map__btf_value_type_id")) + goto out; + + skel->percpu_arr = bpf_map__initial_value(map, &data_sz); + if (!ASSERT_EQ(data_sz, desired_sz, "data_sz")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_arr, "skel->percpu_arr")) + goto out; + if (!ASSERT_EQ(skel->percpu_arr->arr[0], arr_value, "arr[0]")) + goto out; + + err = test_global_percpu_data__load(skel); + if (bpf_map__value_size(map) > pcpu_max_value_size) { + if (ASSERT_EQ(err, -E2BIG, "test_global_percpu_data__load E2BIG")) + /* + * When the PAGE_SIZE is 64KiB, the new value size will + * be 128KiB. Creating a percpu_array map with the new + * value size will fail with -E2BIG due to percpu_array + * map value size limit 64KiB, which is calculated by + * PFN_ALIGN(32KiB). + * Let's skip the test for such case. + */ + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + map = skel->maps.percpu; + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + ASSERT_EQ(skel->bss->arr_sum, arr_value * skel->bss->run, "arr_sum"); + +out: + test_global_percpu_data__destroy(skel); +} + static int create_rdonly_percpu_array(void) { LIBBPF_OPTS(bpf_map_create_opts, map_opts, @@ -388,6 +436,8 @@ void test_global_percpu_data(void) test_global_percpu_data_init(); if (test__start_subtest("lskel")) test_global_percpu_data_lskel(); + if (test__start_subtest("map_resize")) + test_global_percpu_data_map_resize(); if (test__start_subtest("rdonly_direct_read")) test_global_percpu_data_rdonly_direct_read(); if (test__start_subtest("rdonly_direct_write"))
diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c new file mode 100644 index 0000000..dabefdb --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c
@@ -0,0 +1,3155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <endian.h> +#include <stdint.h> +#include <test_progs.h> +#include <bpf/btf.h> +#include <linux/filter.h> + +#include "sysctl_helpers.h" +#include "jit_insns_arena.skel.h" + +#define NR_CASES 512 +#define CTX_SPILL -16 /* stack slot holding the ctx pointer */ +#define MEM_SLOT -8 /* stack slot for the atomic operations */ + +struct sweep_ctx { + __u64 in[NR_CASES][2]; + __u64 out[NR_CASES][4]; +}; + +#define IN_OFF(k, j) (offsetof(struct sweep_ctx, in) + ((k) * 2 + (j)) * 8) +#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 4 + (j)) * 8) + +/* The cases of one program: operands in, results out, expected results */ +struct batch { + struct bpf_insn *insns; + int cnt, cap; + struct sweep_ctx ctx; + __u64 exp[NR_CASES][4]; + __u8 nr_out[NR_CASES]; + int nr; + char desc[80]; +}; + +static char log[1 << 16]; + +#define EMIT(b, insn) ((b)->insns[(b)->cnt++] = (insn)) + +static void emit_ld_imm64(struct batch *b, int reg, __u64 imm) +{ + EMIT(b, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm)); + EMIT(b, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32)); +} + +static void batch_reset(struct batch *b) +{ + b->cnt = 0; + b->nr = 0; + /* The ctx pointer lives on the stack, every register is an operand */ + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, CTX_SPILL)); +} + +static bool batch_init(struct batch *b) +{ + b->cap = NR_CASES * 24 + 8; + b->insns = malloc(b->cap * sizeof(*b->insns)); + if (!ASSERT_OK_PTR(b->insns, "insns")) + return false; + batch_reset(b); + return true; +} + +static void batch_fini(struct batch *b) +{ + free(b->insns); +} + +static int load_prog(struct bpf_insn *insns, int cnt, const char *desc) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", insns, cnt, &lopts); + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +/* Run the batch, check the results, start a new one */ +static void batch_flush(struct batch *b) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &b->ctx, + .ctx_size_in = sizeof(b->ctx), + ); + char buf[160]; + int fd, k, j; + + if (!b->nr) + return; + EMIT(b, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(b, BPF_EXIT_INSN()); + snprintf(buf, sizeof(buf), "%s: load %d cases", b->desc, b->nr); + fd = load_prog(b->insns, b->cnt, buf); + if (fd >= 0 && ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), b->desc)) { + for (k = 0; k < b->nr; k++) { + for (j = 0; j < b->nr_out[k]; j++) { + if (b->ctx.out[k][j] == b->exp[k][j]) + continue; + snprintf(buf, sizeof(buf), "%s: in0=0x%llx in1=0x%llx out%d", + b->desc, b->ctx.in[k][0], b->ctx.in[k][1], j); + ASSERT_EQ(b->ctx.out[k][j], b->exp[k][j], buf); + } + } + } + if (fd >= 0) + close(fd); + batch_reset(b); +} + +/* Add a case: record the operands and load the ctx pointer into rc */ +static int batch_case(struct batch *b, int rc, __u64 in0, __u64 in1) +{ + int k; + + if (b->nr == NR_CASES) + batch_flush(b); + k = b->nr++; + b->ctx.in[k][0] = in0; + b->ctx.in[k][1] = in1; + b->nr_out[k] = 0; + EMIT(b, BPF_LDX_MEM(BPF_DW, rc, BPF_REG_10, CTX_SPILL)); + return k; +} + +static void batch_expect(struct batch *b, int k, int j, __u64 val) +{ + b->exp[k][j] = val; + if (b->nr_out[k] <= j) + b->nr_out[k] = j + 1; +} + +#define LD_IN(b, k, j, rc, reg) EMIT(b, BPF_LDX_MEM(BPF_DW, reg, rc, IN_OFF(k, j))) +#define ST_OUT(b, k, j, rc, reg) EMIT(b, BPF_STX_MEM(BPF_DW, rc, reg, OUT_OFF(k, j))) + +/* A register that is neither of the two operand registers nor r0 */ +static int scratch_reg(int r1, int r2) +{ + int r; + + for (r = BPF_REG_1; r <= BPF_REG_9; r++) + if (r != r1 && r != r2) + return r; + return BPF_REG_9; +} + +static int size_bits(int size) +{ + switch (size) { + case BPF_B: + return 8; + case BPF_H: + return 16; + case BPF_W: + return 32; + default: + return 64; + } +} + +/* The first size bytes of v in memory, as a load of that size reads them */ +static __u64 head_bytes(__u64 v, int size) +{ + int bits = size_bits(size); + + if (bits == 64) + return v; +#if __BYTE_ORDER == __BIG_ENDIAN + return v >> (64 - bits); +#else + return v & ((1ULL << bits) - 1); +#endif +} + +/* v with its first size bytes in memory replaced by those of w */ +static __u64 head_merge(__u64 v, __u64 w, int size) +{ + int bits = size_bits(size); + __u64 mask; + + if (bits == 64) + return w; + mask = (1ULL << bits) - 1; +#if __BYTE_ORDER == __BIG_ENDIAN + return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits)); +#else + return (v & ~mask) | (w & mask); +#endif +} + +/* v as a sign extending load of that size reads it */ +static __s64 sign_extend(__u64 v, int size) +{ + switch (size) { + case BPF_B: + return (__s8)v; + case BPF_H: + return (__s16)v; + case BPF_W: + return (__s32)v; + default: + return v; + } +} + +static const char *alu_name(int op) +{ + switch (BPF_OP(op)) { + case BPF_MOV: return "mov"; + case BPF_NEG: return "neg"; + case BPF_AND: return "and"; + case BPF_OR: return "or"; + case BPF_XOR: return "xor"; + case BPF_LSH: return "lsh"; + case BPF_RSH: return "rsh"; + case BPF_ARSH: return "arsh"; + case BPF_ADD: return "add"; + case BPF_SUB: return "sub"; + case BPF_MUL: return "mul"; + case BPF_DIV: return "div"; + case BPF_MOD: return "mod"; + case BPF_OP(BPF_XCHG): return "xchg"; + case BPF_OP(BPF_CMPXCHG): return "cmpxchg"; + default: return "?"; + } +} + +static const char *jmp_name(int op) +{ + switch (op) { + case BPF_JSET: return "jset"; + case BPF_JEQ: return "jeq"; + case BPF_JNE: return "jne"; + case BPF_JGT: return "jgt"; + case BPF_JGE: return "jge"; + case BPF_JLT: return "jlt"; + case BPF_JLE: return "jle"; + case BPF_JSGT: return "jsgt"; + case BPF_JSGE: return "jsge"; + case BPF_JSLT: return "jslt"; + case BPF_JSLE: return "jsle"; + default: return "?"; + } +} + +/* ALU result computation, with the ISA result for every operand */ +enum { F_ALU32 = 1, F_SIGNED = 2 }; + +static void alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) +{ + bool is_signed = flags & F_SIGNED; + int mask = flags & F_ALU32 ? 31 : 63; + + /* Narrow operands for ALU32 */ + if (flags & F_ALU32) { + if (is_signed) { + v1 = (__u64)(__s32)v1; + v2 = (__u64)(__s32)v2; + } else { + v1 = (__u32)v1; + v2 = (__u32)v2; + } + } + + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_NEG: + *res = -v1; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_LSH: + *res = v1 << (v2 & mask); + break; + case BPF_RSH: + *res = v1 >> (v2 & mask); + break; + case BPF_ARSH: + if (flags & F_ALU32) + v1 = (__u64)(__s32)v1; + v2 &= mask; + *res = v1 >> v2; + if (v2 > 0 && v1 > INT64_MAX) + *res |= ~0ULL << (64 - v2); + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + /* By zero: the quotient is zero and the dividend keeps, by -1: negation and zero */ + case BPF_DIV: + if (v2 == 0) + *res = 0; + else if (!is_signed) + *res = v1 / v2; + else if ((__s64)v2 == -1) + *res = -v1; + else + *res = (__u64)((__s64)v1 / (__s64)v2); + break; + case BPF_MOD: + if (v2 == 0) + *res = v1; + else if (!is_signed) + *res = v1 % v2; + else if ((__s64)v2 == -1) + *res = 0; + else + *res = (__u64)((__s64)v1 % (__s64)v2); + break; + } +} + +/* + * Operand pattern generator for the power-of-two magnitude tests: all + * combinations of magnitudes and signs with a block of contiguous values + * around each magnitude, with every dstep'th magnitude of the destination, + * and a larger block for the same magnitude of both operands. The first + * pattern is where the immediates take every encoding a JIT has, the + * second where the sign and overflow boundaries of both operands meet. + */ +static __s64 value(int msb, int delta, int sign) +{ + return sign * (1LL << msb) + delta; +} + +#define PATTERN_BLOCK1 1 +#define PATTERN_BLOCK2 3 + +typedef void (*emit_fn)(struct batch *b, void *arg, __s64 dst, __s64 src); + +static void fill_pattern(struct batch *b, void *arg, int dbits, int sbits, + int block1, int dstep, int block2, emit_fn emit) +{ + static const int sgn[][2] = { { 1, 1 }, { 1, -1 }, { -1, 1 }, { -1, -1 } }; + int di, si, bt, db, sb, k; + + for (di = 0; di < dbits - 1; di += dstep) + for (si = 0; si < sbits - 1; si++) + for (k = 0; k < ARRAY_SIZE(sgn); k++) + for (db = -(block1 / 2); db < (block1 + 1) / 2; db++) + for (sb = -(block1 / 2); sb < (block1 + 1) / 2; sb++) + emit(b, arg, value(di, db, sgn[k][0]), + value(si, sb, sgn[k][1])); + + for (bt = 0; bt < (dbits > sbits ? dbits : sbits) - 1; bt++) + for (k = 0; k < ARRAY_SIZE(sgn); k++) + for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) + for (sb = -(block2 / 2); sb < (block2 + 1) / 2; sb++) + emit(b, arg, value(bt % dbits, db, sgn[k][0]), + value(bt % sbits, sb, sgn[k][1])); + batch_flush(b); +} + +/* Operands at the edges: zero, one and the limits of every width */ +static const __u64 edges[] = { + 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000, + 0xffffffff, 0x100000000ULL, 0x7fffffffffffffffULL, 0x8000000000000000ULL, + 0xffffffff80000000ULL, 0xffffffff7fffffffULL, 0xffffffffffffffffULL, +}; + +static void fill_edges(struct batch *b, void *arg, bool imm, emit_fn emit) +{ + int i, j; + + for (i = 0; i < ARRAY_SIZE(edges); i++) + for (j = 0; j < ARRAY_SIZE(edges); j++) { + /* The immediate has to fit its 32 bits */ + if (imm && (__s64)edges[j] != (__s32)edges[j]) + continue; + emit(b, arg, edges[i], edges[j]); + } + batch_flush(b); +} + +/* ALU operations with an immediate and with a register for all magnitudes */ +static const int alu_ops[] = { + BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, +}; + +struct alu_arg { + int op; + __u32 flags; +}; + +static void emit_alu_imm(struct batch *b, void *arg, __s64 dst, __s64 imm) +{ + struct alu_arg *a = arg; + bool alu32 = a->flags & F_ALU32; + __u64 res; + int k; + + /* The verifier rejects a zero divisor and a shift count beyond the width */ + if ((a->op == BPF_DIV || a->op == BPF_MOD) && !imm) + return; + if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) && + (__u64)imm >= (alu32 ? 32 : 64)) + return; + if (a->op == BPF_NEG) + imm = 0; + alu_result(&res, dst, (__s32)imm, a->op, a->flags); + k = batch_case(b, BPF_REG_3, dst, (__s32)imm); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_K, + BPF_REG_1, 0, a->flags & F_SIGNED ? 1 : 0, imm)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, alu32 ? (__u32)res : res); +} + +static void emit_alu_reg(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct alu_arg *a = arg; + bool alu32 = a->flags & F_ALU32; + __u64 res; + int k; + + /* Shift counts beyond the width have their own subtest */ + if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) && + (alu32 ? (__u32)src : (__u64)src) >= (alu32 ? 32 : 64)) + return; + alu_result(&res, dst, src, a->op, a->flags); + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_X, + BPF_REG_1, BPF_REG_2, a->flags & F_SIGNED ? 1 : 0, 0)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, alu32 ? (__u32)res : res); +} + +static void test_alu_magnitudes(bool reg) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_ops); o++) { + struct alu_arg a = { alu_ops[o], f ? F_ALU32 : 0 }; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s_%c magnitudes", + f ? 32 : 64, alu_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_alu_reg); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 1, 5, emit_alu_imm); + } + for (o = 0; o < 2; o++) { + struct alu_arg a = { o ? BPF_MOD : BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0) }; + + snprintf(b.desc, sizeof(b.desc), "alu%d_s%s_%c magnitudes", + f ? 32 : 64, alu_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_alu_reg); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 1, 5, emit_alu_imm); + } + } + batch_fini(&b); +} + +static void test_alu_imm(void) +{ + test_alu_magnitudes(false); +} + +static void test_alu_reg(void) +{ + test_alu_magnitudes(true); +} + +static void test_alu_edges(void) +{ + static const struct alu_arg ops[] = { + { BPF_MOV }, { BPF_AND }, { BPF_OR }, { BPF_XOR }, { BPF_LSH }, { BPF_RSH }, + { BPF_ARSH }, { BPF_ADD }, { BPF_SUB }, { BPF_MUL }, { BPF_DIV }, { BPF_MOD }, + { BPF_DIV, F_SIGNED }, { BPF_MOD, F_SIGNED }, { BPF_NEG }, + }; + struct batch b; + int o, f, r, i; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + struct alu_arg a = { ops[o].op, ops[o].flags | (f ? F_ALU32 : 0) }; + + for (r = 0; r < (a.op == BPF_NEG ? 1 : 2); r++) { + snprintf(b.desc, sizeof(b.desc), "alu%d_%s%s_%c edges", + f ? 32 : 64, a.flags & F_SIGNED ? "s" : "", alu_name(a.op), + r ? 'x' : 'k'); + if (a.op == BPF_NEG) { + for (i = 0; i < ARRAY_SIZE(edges); i++) + emit_alu_imm(&b, &a, edges[i], 0); + batch_flush(&b); + } else if (r) { + fill_edges(&b, &a, false, emit_alu_reg); + } else { + fill_edges(&b, &a, true, emit_alu_imm); + } + } + } + } + batch_fini(&b); +} + +/* ALU operations with an immediate for every destination register */ +static const int alu_reg_ops[] = { + BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_LSH, BPF_RSH, BPF_ARSH, + BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, +}; + +static void alu_imm_regs(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32; + __u32 imm = 0x01234567U; + __u64 dst, res; + int rd, rc, k; + + if (flags & F_SIGNED) + dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; + else + dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + imm &= 31; + if (op == BPF_NEG) + imm = 0; + alu_result(&res, dst, imm, op, flags); + if (alu32) + res = (__u32)res; + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_k registers", alu32 ? 32 : 64, + flags & F_SIGNED ? "s" : "", alu_name(op)); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + rc = scratch_reg(rd, rd); + k = batch_case(b, rc, dst, 0); + LD_IN(b, k, 0, rc, rd); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_K, + rd, 0, flags & F_SIGNED ? 1 : 0, imm)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, res); + } + batch_flush(b); +} + +static void test_alu_imm_regs(void) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_reg_ops); o++) + alu_imm_regs(&b, alu_reg_ops[o], f ? F_ALU32 : 0); + alu_imm_regs(&b, BPF_NEG, f ? F_ALU32 : 0); + alu_imm_regs(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + alu_imm_regs(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* + * Division and modulo by constants for every width, signedness and + * destination register with a set of divisors and dividends around the + * multiple boundaries and the extremes of each range. This covers a JIT + * replacing the division with a multiplication by a magic number in all + * of its forms: magic numbers needing the extra add step, zero shifts, + * both signed corrections, and the divisors like 1, -1 and the ones + * above 2^63 that are still emitted as a real division. The dividends + * of the 32-bit operations carry garbage in the upper 32 bits. + */ +static const __s32 unsigned_divisors[] = { + 2, 3, 7, 10, 25, 641, 1000, 4096, 65536, 1000000007, 0x7fffffff, + -2, -10, -0x7fffffff - 1, -1, 1, +}; + +static const __s32 signed_divisors[] = { + 2, 3, 7, 25, 1000, 0x40000000, 0x7fffffff, + -2, -3, -5, -7, -1000, -0x7fffffff, -0x7fffffff - 1, 1, -1, +}; + +static int dividends(__u32 flags, __s32 imm, __u64 *n) +{ + bool alu32 = flags & F_ALU32, is_signed = flags & F_SIGNED; + __u64 d = is_signed ? (imm < 0 ? -(__s64)imm : imm) : + alu32 ? (__u32)imm : (__u64)(__s64)imm; + int cnt = 0, i; + + n[cnt++] = 0; + n[cnt++] = d - 1; + n[cnt++] = d + 1; + n[cnt++] = 3 * d + 2; + if (!alu32) { + n[cnt++] = (d << 40) - 1; + n[cnt++] = (d << 40) + 1; + n[cnt++] = (1ULL << 63) - 1; + n[cnt++] = 1ULL << 63; + n[cnt++] = ~0ULL; + n[cnt++] = 0xdaeb8ebd244a330cULL; + } else { + n[cnt++] = (d << 16) - 1; + n[cnt++] = (d << 16) + 1; + n[cnt++] = (1ULL << 31) - 1; + n[cnt++] = 1ULL << 31; + n[cnt++] = ~0U; + n[cnt++] = 0x8c1e86f1; + } + if (is_signed) { + n[cnt++] = -1; + n[cnt++] = -d - 1; + n[cnt++] = -3 * d - 2; + n[cnt++] = alu32 ? -(1ULL << 31) + 1 : -(1ULL << 63) + 1; + } + if (alu32) + for (i = 0; i < cnt; i++) + n[i] = (__u32)n[i] | ((0xa5a50000ULL + i) << 32); + return cnt; +} + +static __u64 div_result(int op, __u32 flags, __s32 imm, __u64 n) +{ + __u64 res; + + alu_result(&res, n, (__s32)imm, op, flags); + return flags & F_ALU32 ? (__u32)res : res; +} + +static void div_const(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32, is_signed = flags & F_SIGNED; + const __s32 *divs = is_signed ? signed_divisors : unsigned_divisors; + int cnt = is_signed ? ARRAY_SIZE(signed_divisors) : ARRAY_SIZE(unsigned_divisors); + int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_K; + int i, j, nr, rd, rc, k; + __u64 n[16]; + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_k constants", alu32 ? 32 : 64, + is_signed ? "s" : "", alu_name(op)); + for (i = 0; i < cnt; i++) { + nr = dividends(flags, divs[i], n); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + rc = scratch_reg(rd, rd); + for (j = 0; j < nr; j++) { + k = batch_case(b, rc, n[j], divs[i]); + LD_IN(b, k, 0, rc, rd); + EMIT(b, BPF_RAW_INSN(code, rd, 0, is_signed, divs[i])); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, div_result(op, flags, divs[i], n[j])); + } + } + } + batch_flush(b); +} + +static void test_div_const(void) +{ + struct batch b; + int f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + div_const(&b, BPF_DIV, f ? F_ALU32 : 0); + div_const(&b, BPF_MOD, f ? F_ALU32 : 0); + div_const(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + div_const(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* ALU operations for all combinations of operand registers */ +static void alu_reg_pairs(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32, divmod = op == BPF_DIV || op == BPF_MOD; + int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X; + int off = flags & F_SIGNED ? 1 : 0, rd, rs, rc, k, n, nr = 1; + __u64 dst, srcs[3], src, res, same; + + if (flags & F_SIGNED) + dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; + else + dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; + srcs[0] = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL; + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + srcs[0] &= 31; + /* A zero divisor, in 32 bits also one with only the upper half set */ + if (divmod) { + srcs[nr++] = 0; + if (alu32) + srcs[nr++] = 1ULL << 32; + } + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_x register pairs", alu32 ? 32 : 64, + flags & F_SIGNED ? "s" : "", alu_name(op)); + for (n = 0; n < nr; n++) { + src = srcs[n]; + alu_result(&res, dst, src, op, flags); + alu_result(&same, src, src, op, flags); + if (alu32) { + res = (__u32)res; + same = (__u32)same; + } + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, rd == rs ? src : dst, src); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_RAW_INSN(code, rd, rs, off, 0)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, rd == rs ? same : res); + } + } + } + batch_flush(b); +} + +static void test_alu_reg_pairs(void) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_reg_ops); o++) + alu_reg_pairs(&b, alu_reg_ops[o], f ? F_ALU32 : 0); + alu_reg_pairs(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + alu_reg_pairs(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +static const __s64 shift_vals[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dword < 0, word > 0 */ + 0xfedcba0198765432LL, /* dword < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ +}; +static const int shift_ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH }; + +/* Shifts for all shift counts, by immediate, by register and by the shifted register */ +static void test_shifts(void) +{ + struct batch b; + int o, f, v, imm, k; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + int bits = f ? 32 : 64; + + for (o = 0; o < ARRAY_SIZE(shift_ops); o++) { + int op = shift_ops[o]; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s all shift counts", bits, + alu_name(op)); + for (v = 0; v < ARRAY_SIZE(shift_vals); v++) { + for (imm = 0; imm < bits; imm++) { + __u64 reg = shift_vals[v], res; + + if (f) + reg = op == BPF_ARSH ? (__u64)(__s32)reg : + (__u32)reg; + alu_result(&res, reg, imm, op, 0); + if (f) + res = (__u32)res; + + k = batch_case(&b, BPF_REG_3, shift_vals[v], imm); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + if (f) + EMIT(&b, BPF_ALU32_IMM(op, BPF_REG_1, imm)); + else + EMIT(&b, BPF_ALU64_IMM(op, BPF_REG_1, imm)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, res); + + k = batch_case(&b, BPF_REG_3, shift_vals[v], imm); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, res); + } + } + /* The count in the shifted register */ + for (imm = 0; imm < bits; imm++) { + __u64 res; + + alu_result(&res, imm, imm, op, 0); + k = batch_case(&b, BPF_REG_3, imm, 0); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_1)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_1)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, f ? (__u32)res : res); + } + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* + * Shifts by register with counts at and above the width and negative ones, + * which the ISA masks by the width. Apart in a subtest of its own, as not + * every JIT does so. + */ +static void test_shift_counts(void) +{ + static const __s64 counts[] = { 32, 33, 63, 64, 65, 127, -1, -3, 1LL << 32 | 3 }; + struct batch b; + int o, f, v, c, k; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + int bits = f ? 32 : 64; + + for (o = 0; o < ARRAY_SIZE(shift_ops); o++) { + int op = shift_ops[o]; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s_x counts beyond the width", + bits, alu_name(op)); + for (c = 0; c < ARRAY_SIZE(counts); c++) { + __u64 cnt = f ? (__u32)counts[c] : counts[c]; + + for (v = 0; v < ARRAY_SIZE(shift_vals); v++) { + __u64 reg = shift_vals[v], res; + + if (f) + reg = op == BPF_ARSH ? (__u64)(__s32)reg : + (__u32)reg; + alu_result(&res, reg, cnt & (bits - 1), op, 0); + k = batch_case(&b, BPF_REG_3, shift_vals[v], counts[c]); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, f ? (__u32)res : res); + } + } + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* Atomic operations on the stack for all magnitudes of memory and source operand */ +static const int atomic_ops[] = { + BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, + BPF_ADD | BPF_FETCH, BPF_AND | BPF_FETCH, BPF_OR | BPF_FETCH, BPF_XOR | BPF_FETCH, + BPF_XCHG, +}; + +struct atomic_arg { + int op; + int size; +}; + +static void emit_atomic(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct atomic_arg *a = arg; + bool w = a->size == BPF_W; + __u64 keep = 0x0123456789abcdefULL, res, fetch; + int k; + + if (a->op == BPF_XCHG) + res = src; + else + alu_result(&res, w ? (__u32)dst : dst, w ? (__u32)src : src, BPF_OP(a->op), 0); + /* The source register gets the old value with fetch, keeps its own without */ + fetch = a->op & BPF_FETCH ? (w ? (__u32)dst : dst) : src; + + k = batch_case(b, BPF_REG_3, dst, src); + /* The kept value fills the slot, a 32-bit operation leaves the other half alone */ + emit_ld_imm64(b, BPF_REG_0, keep); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_STX_MEM(a->size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_ATOMIC_OP(a->size, a->op, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_2); + ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + batch_expect(b, k, 0, head_merge(keep, w ? (__u32)res : res, a->size)); + batch_expect(b, k, 1, fetch); + batch_expect(b, k, 2, keep); +} + +static void emit_cmpxchg(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + int size = *(int *)arg; + bool w = size == BPF_W; + __u64 old = w ? (__u32)dst : dst; + int k; + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + /* The inverted value fills the slot, a 32-bit operation leaves the other half alone */ + EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1)); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + /* Mismatch: the memory keeps its value, r0 gets it */ + EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_0); + /* + * Match: the memory gets the source, r0 the old value. r0 holds the + * full value, so the 32-bit compare has to ignore its upper half, and + * the source register has to keep its own. + */ + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_0); + EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_1); + ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + ST_OUT(b, k, 3, BPF_REG_3, BPF_REG_2); + batch_expect(b, k, 0, old); + batch_expect(b, k, 1, head_merge(~(__u64)dst, w ? (__u32)src : src, size)); + batch_expect(b, k, 2, old); + batch_expect(b, k, 3, src); +} + +static void test_atomics(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + struct batch b; + int o, s; + + if (!batch_init(&b)) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (o = 0; o < ARRAY_SIZE(atomic_ops); o++) { + struct atomic_arg a = { atomic_ops[o], size }; + + snprintf(b.desc, sizeof(b.desc), "atomic%d_%s%s magnitudes", + size == BPF_DW ? 64 : 32, alu_name(a.op), + a.op == BPF_XCHG ? "" : a.op & BPF_FETCH ? "_fetch" : ""); + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_atomic); + } + snprintf(b.desc, sizeof(b.desc), "atomic%d_cmpxchg magnitudes", + size == BPF_DW ? 64 : 32); + fill_pattern(&b, &size, 64, 64, 0, 1, PATTERN_BLOCK2, emit_cmpxchg); + } + batch_fini(&b); +} + +/* Atomic operations for all combinations of pointer and source register */ +static void atomic_reg_pairs(struct batch *b, int size, int op) +{ + bool w = size == BPF_W; + __u64 mem, upd; + int rd, rs, rc, k, n; + + if (w) { + mem = 0xfedcba9801234567ULL; + upd = 0x76543210U; + } else { + mem = 0x0123456789abcdefULL; + upd = 0xfedcba9876543210ULL; + } + + snprintf(b->desc, sizeof(b->desc), "atomic%d_%s%s register pairs", w ? 32 : 64, + alu_name(op), op == BPF_XCHG || op == BPF_CMPXCHG ? "" : + op & BPF_FETCH ? "_fetch" : ""); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + __u64 regs[10], old = w ? (__u32)mem : mem, new, src, cmp; + + /* The source would be the pointer, r0 the compare value */ + if (rs == rd || (op == BPF_CMPXCHG && rd == BPF_REG_0)) + continue; + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, mem, upd); + /* r0 inverted into the slot, memory from in0, source from in1, pointer */ + LD_IN(b, k, 0, rc, BPF_REG_0); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + LD_IN(b, k, 0, rc, rs); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT)); + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_MOV64_REG(rd, BPF_REG_10)); + EMIT(b, BPF_ATOMIC_OP(size, op, rd, rs, MEM_SLOT)); + /* r0 holds the pointer when it is the destination */ + n = 0; + if (rd != BPF_REG_0) + ST_OUT(b, k, n++, rc, BPF_REG_0); + ST_OUT(b, k, n++, rc, rs); + EMIT(b, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, n, rc, rs); + + /* Model with the register aliasing of r0 */ + regs[BPF_REG_0] = ~mem; + regs[rs] = upd; + src = w ? (__u32)regs[rs] : regs[rs]; + if (op == BPF_XCHG) { + new = src; + regs[rs] = old; + } else if (op == BPF_CMPXCHG) { + cmp = w ? (__u32)regs[BPF_REG_0] : regs[BPF_REG_0]; + new = cmp == old ? src : old; + regs[BPF_REG_0] = old; + } else { + alu_result(&new, old, src, BPF_OP(op), 0); + if (w) + new = (__u32)new; + if (op & BPF_FETCH) + regs[rs] = old; + } + n = 0; + if (rd != BPF_REG_0) + batch_expect(b, k, n++, regs[BPF_REG_0]); + batch_expect(b, k, n++, regs[rs]); + batch_expect(b, k, n, head_merge(~mem, new, size)); + } + } + batch_flush(b); +} + +static void test_atomic_reg_pairs(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + struct batch b; + int o, s; + + if (!batch_init(&b)) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + for (o = 0; o < ARRAY_SIZE(atomic_ops); o++) + atomic_reg_pairs(&b, sizes[s], atomic_ops[o]); + atomic_reg_pairs(&b, sizes[s], BPF_CMPXCHG); + } + batch_fini(&b); +} + +/* A maximal chain of atomic adds on one stack slot */ +static void test_xadd_chain(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_insn *insns; + int len = BPF_MAXINSNS, s, i, fd; + + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + insns[0] = BPF_MOV32_IMM(BPF_REG_0, 1); + insns[1] = BPF_ST_MEM(sizes[s], BPF_REG_10, -40, 42); + for (i = 2; i < len - 2; i++) + insns[i] = BPF_STX_XADD(sizes[s], BPF_REG_10, BPF_REG_0, -40); + insns[len - 2] = BPF_LDX_MEM(sizes[s], BPF_REG_0, BPF_REG_10, -40); + insns[len - 1] = BPF_EXIT_INSN(); + + fd = load_prog(insns, len, "xadd chain"); + if (fd < 0) + break; + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "xadd chain run")) + ASSERT_EQ(opts.retval, 42 + len - 4, "xadd chain result"); + close(fd); + } + free(insns); +} + +/* 64-bit immediate loads for all magnitudes, byte patterns and registers */ +static void ld_imm64_case(struct batch *b, int rd, __u64 imm) +{ + int rc = scratch_reg(rd, rd); + int k = batch_case(b, rc, imm, 0); + + emit_ld_imm64(b, rd, imm); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, imm); +} + +static void test_ld_imm64(void) +{ + static const struct { + __u8 base1, mask1, base2, mask2; + } bytes[] = { + { 0, 0xff, 0xff, 0xff }, /* checker */ + { 1, 0x81, 0x80, 0x80 }, /* positive and negative */ + { 1, 0x81, 0, 0xff }, /* positive and zero */ + { 0x80, 0x80, 0, 0xff }, /* negative and zero */ + }; + /* Zero, a 32-bit, a sign extended 32-bit and a full immediate for every register */ + static const __u64 imms[] = { + 0, 1, 0x7fffffff, 0x80000000, 0xffffffff, 0x100000000ULL, + 0xffffffffffffffffULL, 0xffffffff80000000ULL, 0xffffffff7fffffffULL, + 0x0123456789abcdefULL, + }; + int block = 64, bit, adj, sign, p, pattern, index, rd, i; + struct batch b; + + if (!batch_init(&b)) + return; + snprintf(b.desc, sizeof(b.desc), "ld_imm64 magnitudes"); + for (bit = 0; bit <= 62; bit++) + for (adj = -block / 2; adj < block / 2; adj++) + for (sign = -1; sign <= 1; sign += 2) + ld_imm64_case(&b, BPF_REG_1, sign * ((1LL << bit) + adj)); + batch_flush(&b); + + for (p = 0; p < ARRAY_SIZE(bytes); p++) { + __u32 rand = 1; + + snprintf(b.desc, sizeof(b.desc), "ld_imm64 byte pattern %d", p); + for (pattern = 0; pattern < 256; pattern++) { + __u64 imm = 0; + + for (index = 0; index < 8; index++) { + int byte; + + if (pattern & (1 << index)) + byte = (bytes[p].base1 & bytes[p].mask1) | + (rand & ~bytes[p].mask1); + else + byte = (bytes[p].base2 & bytes[p].mask2) | + (rand & ~bytes[p].mask2); + imm = (imm << 8) | (__u8)byte; + } + rand = rand * 1664525 + 1013904223; + ld_imm64_case(&b, BPF_REG_1, imm); + } + batch_flush(&b); + } + snprintf(b.desc, sizeof(b.desc), "ld_imm64 registers"); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) + for (i = 0; i < ARRAY_SIZE(imms); i++) + ld_imm64_case(&b, rd, imms[i]); + batch_flush(&b); + batch_fini(&b); +} + +/* Conditional jumps with an immediate and with a register for all magnitudes */ +static const int jmp_ops[] = { + BPF_JSET, BPF_JEQ, BPF_JNE, BPF_JGT, BPF_JGE, BPF_JLT, BPF_JLE, + BPF_JSGT, BPF_JSGE, BPF_JSLT, BPF_JSLE, +}; + +static bool jmp_match(__s64 v1, __s64 v2, __u8 op) +{ + switch (op) { + case BPF_JSET: + return !!(v1 & v2); + case BPF_JEQ: + return v1 == v2; + case BPF_JNE: + return v1 != v2; + case BPF_JGT: + return (__u64)v1 > (__u64)v2; + case BPF_JGE: + return (__u64)v1 >= (__u64)v2; + case BPF_JLT: + return (__u64)v1 < (__u64)v2; + case BPF_JLE: + return (__u64)v1 <= (__u64)v2; + case BPF_JSGT: + return v1 > v2; + case BPF_JSGE: + return v1 >= v2; + case BPF_JSLT: + return v1 < v2; + case BPF_JSLE: + return v1 <= v2; + } + return false; +} + +struct jmp_arg { + int op; + bool is32; + bool reg; +}; + +static void emit_jmp(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct jmp_arg *a = arg; + bool match; + int k; + + if (a->is32) + match = jmp_match((__s32)dst, (__s32)src, a->op); + else + match = jmp_match(dst, a->reg ? src : (__s32)src, a->op); + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + if (a->reg) + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_MOV64_IMM(BPF_REG_4, 1)); + if (a->reg && a->is32) + EMIT(b, BPF_JMP32_REG(a->op, BPF_REG_1, BPF_REG_2, 1)); + else if (a->reg) + EMIT(b, BPF_JMP_REG(a->op, BPF_REG_1, BPF_REG_2, 1)); + else if (a->is32) + EMIT(b, BPF_JMP32_IMM(a->op, BPF_REG_1, src, 1)); + else + EMIT(b, BPF_JMP_IMM(a->op, BPF_REG_1, src, 1)); + EMIT(b, BPF_MOV64_IMM(BPF_REG_4, 0)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_4); + batch_expect(b, k, 0, match); +} + +static void test_jmp_magnitudes(bool reg) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + struct jmp_arg a = { jmp_ops[o], f, reg }; + + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c magnitudes", f ? "32" : "", + jmp_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_jmp); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 8, PATTERN_BLOCK2, + emit_jmp); + } + } + batch_fini(&b); +} + +static void test_jmp_imm(void) +{ + test_jmp_magnitudes(false); +} + +static void test_jmp_reg(void) +{ + test_jmp_magnitudes(true); +} + +static void test_jmp_edges(void) +{ + struct batch b; + int o, f, r; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + for (r = 0; r < 2; r++) { + struct jmp_arg a = { jmp_ops[o], f, r }; + + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c edges", f ? "32" : "", + jmp_name(a.op), r ? 'x' : 'k'); + fill_edges(&b, &a, !r, emit_jmp); + } + } + } + batch_fini(&b); +} + +/* A conditional jump on two registers, with the operands swapped for n == 1 */ +static void jmp_reg_pair(struct batch *b, bool is32, int op, int rd, int rs, int n) +{ + const __u64 a = 0x7edcba9876543210ULL, c = 0x0123456789abcdefULL; + __u64 v1 = n ? c : a, v2 = n ? a : c; + int rc, rt, k; + bool match; + + /* Two scratch registers besides the operands */ + rc = scratch_reg(rd, rs); + for (rt = BPF_REG_1; rt < BPF_REG_9; rt++) + if (rt != rd && rt != rs && rt != rc) + break; + if (rd == rs) + v2 = v1; + match = is32 ? jmp_match((__s32)v1, (__s32)v2, op) : jmp_match(v1, v2, op); + + k = batch_case(b, rc, v1, v2); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_MOV64_IMM(rt, 1)); + if (is32) + EMIT(b, BPF_JMP32_REG(op, rd, rs, 1)); + else + EMIT(b, BPF_JMP_REG(op, rd, rs, 1)); + EMIT(b, BPF_MOV64_IMM(rt, 0)); + ST_OUT(b, k, 0, rc, rt); + batch_expect(b, k, 0, match); +} + +/* Conditional jumps for all combinations of operand registers, taken and not */ +static void test_jmp_reg_pairs(void) +{ + struct batch b; + int o, f, rd, rs, n; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_x register pairs", + f ? "32" : "", jmp_name(jmp_ops[o])); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) + for (n = 0; n < 2; n++) + jmp_reg_pair(&b, f, jmp_ops[o], rd, rs, n); + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* The offset of a jump instruction, which the 32-bit unconditional jump keeps in imm */ +static void set_jmp_off(struct bpf_insn *insn, bool ja32, int off) +{ + if (ja32) + insn->imm = off; + else + insn->off = off; +} + +static struct bpf_insn jmp_insn(bool ja32) +{ + return ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, 0) : BPF_JMP_IMM(BPF_JA, 0, 0, 0); +} + +/* + * Taken jumps over regions of doubling length, forward or backward, up to + * the maximal offset of the jump: 16 bits, or 2^17 for the 32-bit + * unconditional jump. The operands are reloaded in every region, so that + * the verifier cannot decide the jumps from the bounds of the region + * before and fold the regions away. r0 gains the region number before the + * jump and loses it again only on the target, so the not taken path of a + * conditional jump, the guard of an unconditional one and a jump landing + * short of its target, or forward past it, exit with a nonzero r0. A + * backward jump overshooting its target comes back to itself and hangs + * the run instead. + */ +static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool backward, + const char *desc) +{ + bool ja32 = BPF_CLASS(jmp->code) == BPF_JMP32 && BPF_OP(jmp->code) == BPF_JA; + bool ja = BPF_OP(jmp->code) == BPF_JA; + struct { __u64 r1, r2; } ctx = { r1, r2 }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + int max = ja32 ? 1 << 17 : INT16_MAX; + int len = 256 + 2 * max, i = 0, n = 0, gap, fd; + struct bpf_insn *insns; + + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + insns[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); + for (gap = backward ? 4 : 2; gap <= max; gap = gap < max / 2 ? gap * 2 : max) { + int k, a, t, x, g, j; + + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_6, 8); + insns[i++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, ++n); + if (backward) { + /* + * Over the region to its guard and jump, which comes back + * to the start of the region, from where it leads on to + * the next one. + */ + a = i++; + t = i; + insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n); + for (k = 0; k < gap - 3 - ja; k++) + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); + x = i++; + g = i; + if (ja) + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); + j = i; + insns[i] = *jmp; + set_jmp_off(&insns[i++], ja32, t - (j + 1)); + insns[i++] = BPF_EXIT_INSN(); + insns[a] = jmp_insn(ja32); + set_jmp_off(&insns[a], ja32, g - (a + 1)); + insns[x] = jmp_insn(ja32); + set_jmp_off(&insns[x], ja32, i - (x + 1)); + } else { + /* An unconditional jump needs a way into its region */ + if (ja) + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); + insns[i] = *jmp; + set_jmp_off(&insns[i++], ja32, gap); + for (k = 0; k < gap - 1; k++) + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); + insns[i++] = BPF_EXIT_INSN(); + insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n); + } + if (gap == max) + break; + } + insns[i++] = BPF_EXIT_INSN(); + if (!ASSERT_LE(i, len, "insn count")) + goto out; + + fd = load_prog(insns, i, desc); + if (fd >= 0) { + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(opts.retval, 0, desc); + close(fd); + } +out: + free(insns); +} + +static void test_staggered_jumps(void) +{ + /* A representative set, with operands which make the jump taken */ + static const struct { + const char *desc; + struct bpf_insn jmp; + __s64 r1, r2; + } kinds[] = { + { "staggered ja", + BPF_JMP_IMM(BPF_JA, 0, 0, 0), 0, 1 }, + { "staggered gotol", + BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, 0), 0, 1 }, + { "staggered jmp_jeq_k", + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 1234, 0), 1234, 0 }, + { "staggered jmp_jne_x", + BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_2, 0), 4321, 1234 }, + { "staggered jmp_jset_k", + BPF_JMP_IMM(BPF_JSET, BPF_REG_1, 0x82, 0), 0x86, 0 }, + { "staggered jmp_jsgt_x", + BPF_JMP_REG(BPF_JSGT, BPF_REG_1, BPF_REG_2, 0), -1, -2 }, + { "staggered jmp32_jlt_k", + BPF_JMP32_IMM(BPF_JLT, BPF_REG_1, 0x80000000, 0), 1234, 0 }, + { "staggered jmp32_jsge_x", + BPF_JMP32_REG(BPF_JSGE, BPF_REG_1, BPF_REG_2, 0), -2, -2 }, + }; + char desc[64]; + int i, d; + + for (i = 0; i < ARRAY_SIZE(kinds); i++) { + for (d = 0; d < 2; d++) { + snprintf(desc, sizeof(desc), "%s %s", kinds[i].desc, + d ? "backward" : "forward"); + staggered_jumps(&kinds[i].jmp, kinds[i].r1, kinds[i].r2, d, desc); + } + } +} + +/* + * The pattern sweeps: one program per pattern and register choice, run + * once per input with the operands in the ctx and the result handed back + * through it. + */ +struct pat_ctx { + __u64 in0; + __u64 in1; + __u64 res; +}; + +#define IN0 offsetof(struct pat_ctx, in0) +#define IN1 offsetof(struct pat_ctx, in1) +#define RES offsetof(struct pat_ctx, res) + +/* r6 holds the ctx */ +static const int pat_regs[] = { + BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, + BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, +}; + +/* The second register of a pattern: the same one, the next and a far one */ +#define NR_PAIRS 3 +static const int pair_off[NR_PAIRS] = { 0, 1, 5 }; + +static int pair_reg(int i, int k) +{ + return pat_regs[(i + pair_off[k]) % ARRAY_SIZE(pat_regs)]; +} + +static const __u64 inputs[] = { + 0, 0x1122334455667788ULL, 0xfedcba9876543210ULL, 0x80, 0xffffffffffffff7fULL, +}; + +struct prog { + struct bpf_insn insns[16]; + int cnt; +}; + +static void prog_start(struct prog *p) +{ + p->cnt = 0; + EMIT(p, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); +} + +/* Hand the result in reg back and return */ +static void prog_end(struct prog *p, int reg) +{ + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_6, reg, RES)); + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(p, BPF_EXIT_INSN()); +} + +static int prog_load(struct prog *p, const char *desc) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p->insns, p->cnt, &lopts); + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +static void prog_run(int fd, __u64 in0, __u64 in1, __u64 expected, const char *desc) +{ + struct pat_ctx ctx = { .in0 = in0, .in1 = in1 }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + char buf[128]; + + snprintf(buf, sizeof(buf), "%s in0=0x%llx in1=0x%llx", desc, in0, in1); + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), buf)) + ASSERT_EQ(ctx.res, expected, buf); +} + +static void test_ldst_zero_off(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + static const int aops[] = { + BPF_ADD, BPF_ADD | BPF_FETCH, BPF_OR, BPF_XCHG, BPF_CMPXCHG, + }; + static const __s32 imms[] = { 0x5a, -1 }; + struct prog p; + char desc[64]; + int b, k, s, j, fd; + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + /* Load with a zero offset over all ones, also into the base */ + prog_start(&p); + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); + EMIT(&p, BPF_LDX_MEM(size, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "ldx size=%d base=r%d dst=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, + head_bytes(inputs[j], size), desc); + close(fd); + + /* Sign extending load with a zero offset from the stack */ + if (size != BPF_DW) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | size, + rd, rb, 0, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), + "ldsx size=%d base=r%d dst=r%d", size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = head_bytes(inputs[j], size); + + v = sign_extend(v, size); + prog_run(fd, inputs[j], 0, v, desc); + } + close(fd); + } + + if (rd == rb) + continue; + /* Store with a zero offset over in0 */ + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_STX_MEM(size, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "stx size=%d base=r%d src=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], in1, size), desc); + } + close(fd); + } + + /* Store an immediate with a zero offset into a zeroed stack slot */ + for (k = 0; k < ARRAY_SIZE(imms); k++) { + int rd = pair_reg(b, 1); + + prog_start(&p); + EMIT(&p, BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ST_MEM(size, rb, 0, imms[k])); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "st size=%d base=r%d imm=%d", + size, rb, imms[k]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, + head_merge(0, (__u64)(__s64)imms[k], size), desc); + close(fd); + } + + /* Atomics with a zero offset on a stack slot holding in0 */ + for (k = 0; k < ARRAY_SIZE(aops); k++) { + int rd = pair_reg(b, 1), op = aops[k]; + + if (size == BPF_B || size == BPF_H) + break; + /* cmpxchg compares with r0 */ + if (op == BPF_CMPXCHG && (rb == BPF_REG_0 || rd == BPF_REG_0)) + continue; + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + if (op == BPF_CMPXCHG) + EMIT(&p, BPF_LDX_MEM(size, BPF_REG_0, BPF_REG_10, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, op, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "atomic 0x%x size=%d base=r%d src=r%d", + op, size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + __u64 v = head_bytes(inputs[j], size); + /* The source is the low half, whichever the byte order */ + __u64 w = size == BPF_W ? (__u32)in1 : in1; + __u64 r = BPF_OP(op) == BPF_ADD ? v + w : + BPF_OP(op) == BPF_OR ? v | w : w; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], r, size), desc); + } + close(fd); + } + } + } +} + +/* + * Loads, stores and stores of an immediate with the offsets around the + * limits of the short displacement forms, negative, large and unaligned + * ones, with every register as base. The memory is an array map value with + * the base register pointing at its middle, the loads hand their results + * back through a second value and the stores are checked against a copy + * kept here. The unaligned offsets are left out where the verifier insists + * on aligned accesses. + */ +#define MEM_SIZE 65536 +#define MEM_BASE (MEM_SIZE / 2) +#define MEM_KEY 0 +#define RES_KEY 1 +#define SLOT_KEY -4 /* stack slots: map key, results and base pointer */ +#define SLOT_RES -16 +#define SLOT_BASE -24 +#define STORE_VAL 0x8899aabbccddeeffULL + +struct off_prog { + struct bpf_insn insns[128]; + int cnt; +}; + +/* The map holding the memory and the results, a copy of the memory and a buffer */ +struct off_mem { + int map_fd; + __u8 *mem, *exp, *res; +}; + +enum { K_LDX, K_LDSX, K_STX, K_ST }; + +/* The offsets for an access of size, aligned unless unaligned ones are ok */ +static int offsets_for(int size, bool unaligned, int *off) +{ + int n = size_bits(size) / 8, cnt = 0; + + off[cnt++] = n; + off[cnt++] = 128 - n; + off[cnt++] = 128; + off[cnt++] = 256 - n; + off[cnt++] = 256; + off[cnt++] = 4096; + off[cnt++] = MEM_BASE - n; + off[cnt++] = -n; + off[cnt++] = -128; + off[cnt++] = -128 - n; + off[cnt++] = -256; + off[cnt++] = -4096; + off[cnt++] = -MEM_BASE; + if (n > 1 && unaligned) { + off[cnt++] = 13; + off[cnt++] = 127; + off[cnt++] = -13; + off[cnt++] = -129; + } + return cnt; +} + +/* The bytes a store of size writes from the register value v */ +static void store_bytes(__u8 *dst, __u64 v, int size) +{ + int n = size_bits(size) / 8; + +#if __BYTE_ORDER == __BIG_ENDIAN + memcpy(dst, (__u8 *)&v + 8 - n, n); +#else + memcpy(dst, &v, n); +#endif +} + +/* The value a load of size reads from the bytes at src */ +static __u64 load_bytes(const __u8 *src, int size) +{ + __u64 v = 0; + + memcpy(&v, src, size_bits(size) / 8); + return head_bytes(v, size); +} + +/* A register that is neither of the two given ones */ +static int other_reg(int r1, int r2) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pat_regs); i++) + if (pat_regs[i] != r1 && pat_regs[i] != r2) + return pat_regs[i]; + return BPF_REG_9; +} + +static void off_prog_ld_imm64(struct off_prog *p, int reg, __u64 imm) +{ + EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm)); + EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32)); +} + +/* Look up the value of key into r0, exit with 1 when the lookup fails */ +static void off_prog_lookup(struct off_prog *p, int map_fd, int key) +{ + EMIT(p, BPF_ST_MEM(BPF_W, BPF_REG_10, SLOT_KEY, key)); + EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, BPF_PSEUDO_MAP_FD, 0, map_fd)); + EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, 0)); + EMIT(p, BPF_MOV64_REG(BPF_REG_2, BPF_REG_10)); + EMIT(p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, SLOT_KEY)); + EMIT(p, BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem)); + EMIT(p, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2)); + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 1)); + EMIT(p, BPF_EXIT_INSN()); +} + +/* rb points at the middle of the memory, the results value is spilled */ +static void off_prog_start(struct off_prog *p, int map_fd, int rb) +{ + p->cnt = 0; + off_prog_lookup(p, map_fd, RES_KEY); + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, SLOT_RES)); + off_prog_lookup(p, map_fd, MEM_KEY); + EMIT(p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, MEM_BASE)); + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, SLOT_BASE)); + if (rb != BPF_REG_0) + EMIT(p, BPF_MOV64_REG(rb, BPF_REG_0)); +} + +static int off_prog_load(struct off_prog *p, const char *desc, bool quiet) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(p, BPF_EXIT_INSN()); + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p->insns, p->cnt, &lopts); + if (quiet) + return fd; + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +static int off_prog_run(int fd, const char *desc) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(fd, &opts); + close(fd); + if (!ASSERT_OK(err, desc) || !ASSERT_EQ(opts.retval, 0, desc)) + return -1; + return 0; +} + +/* Whether the verifier accepts a halfword load from an odd offset */ +static bool unaligned_ok(int map_fd) +{ + struct off_prog p; + int fd; + + off_prog_start(&p, map_fd, BPF_REG_1); + EMIT(&p, BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 1)); + fd = off_prog_load(&p, "unaligned probe", true); + if (fd < 0) + return false; + close(fd); + return true; +} + +/* One program loading from all the offsets of size through rb into rd */ +static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd, + const int *off, int nr) +{ + __u32 mem_key = MEM_KEY, res_key = RES_KEY; + int rs = other_reg(rb, rd), fd, i; + __u64 *res = (__u64 *)m->res; + int map_fd = m->map_fd; + struct off_prog p; + char desc[64]; + + off_prog_start(&p, map_fd, rb); + for (i = 0; i < nr; i++) { + /* The base gets reloaded when it is the destination, all ones otherwise */ + if (rd == rb) + EMIT(&p, BPF_LDX_MEM(BPF_DW, rb, BPF_REG_10, SLOT_BASE)); + else + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + if (kind == K_LDX) + EMIT(&p, BPF_LDX_MEM(size, rd, rb, off[i])); + else + EMIT(&p, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | size, rd, rb, off[i], 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, SLOT_RES)); + EMIT(&p, BPF_STX_MEM(BPF_DW, rs, rd, i * 8)); + } + snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d", + kind == K_LDX ? "ldx" : "ldsx", size, rb, rd); + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &mem_key, m->mem, 0), desc)) + return; + fd = off_prog_load(&p, desc, false); + if (fd < 0) + return; + if (off_prog_run(fd, desc) || !ASSERT_OK(bpf_map_lookup_elem(map_fd, &res_key, res), desc)) + return; + for (i = 0; i < nr; i++) { + __u64 v = load_bytes(m->mem + MEM_BASE + off[i], size); + + if (kind == K_LDSX) + v = sign_extend(v, size); + if (res[i] != v) { + snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d off=%d", + kind == K_LDX ? "ldx" : "ldsx", size, rb, rd, off[i]); + ASSERT_EQ(res[i], v, desc); + } + } +} + +/* One program storing rd, or the immediate imm when rd < 0, at all the offsets */ +static void store_offsets(struct off_mem *m, int size, int rb, int rd, __s32 imm, + const int *off, int nr) +{ + __u64 val = rd >= 0 ? STORE_VAL : (__u64)(__s64)imm; + __u8 *mem = m->mem, *exp = m->exp, *res = m->res; + __u32 mem_key = MEM_KEY; + int map_fd = m->map_fd; + char desc[64], buf[96]; + struct off_prog p; + int fd, i; + + off_prog_start(&p, map_fd, rb); + if (rd >= 0) { + EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, rd, 0, 0, (__u32)STORE_VAL)); + EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, STORE_VAL >> 32)); + } + memcpy(exp, mem, MEM_SIZE); + for (i = 0; i < nr; i++) { + if (rd >= 0) + EMIT(&p, BPF_STX_MEM(size, rb, rd, off[i])); + else + EMIT(&p, BPF_ST_MEM(size, rb, off[i], imm)); + store_bytes(exp + MEM_BASE + off[i], val, size); + } + if (rd >= 0) + snprintf(desc, sizeof(desc), "stx size=%d base=r%d src=r%d", size, rb, rd); + else + snprintf(desc, sizeof(desc), "st size=%d base=r%d imm=%d", size, rb, imm); + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &mem_key, mem, 0), desc)) + return; + fd = off_prog_load(&p, desc, false); + if (fd < 0) + return; + if (off_prog_run(fd, desc) || !ASSERT_OK(bpf_map_lookup_elem(map_fd, &mem_key, res), desc)) + return; + for (i = 0; i < MEM_SIZE && res[i] == exp[i]; i++) + ; + if (i < MEM_SIZE) { + snprintf(buf, sizeof(buf), "%s: byte at offset %d", desc, i - MEM_BASE); + ASSERT_EQ(res[i], exp[i], buf); + } +} + +static void test_ldst_offsets(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + static const __s32 imms[] = { -1, 0x7fffffff, -0x7fffffff - 1 }; + int b, k, s, i, nr, off[32]; + struct off_mem m = {}; + bool unaligned; + + m.map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "ldst_mem", sizeof(__u32), MEM_SIZE, 2, NULL); + if (!ASSERT_GE(m.map_fd, 0, "ldst_mem map")) + return; + m.mem = malloc(MEM_SIZE); + m.exp = malloc(MEM_SIZE); + m.res = malloc(MEM_SIZE); + if (!ASSERT_OK_PTR(m.mem, "mem") || !ASSERT_OK_PTR(m.exp, "exp") || + !ASSERT_OK_PTR(m.res, "res")) + goto out; + for (i = 0; i < MEM_SIZE; i++) + m.mem[i] = (i * 7 + 3) ^ (i >> 8); + unaligned = unaligned_ok(m.map_fd); + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int rb = pat_regs[b], size = sizes[s]; + + nr = offsets_for(size, unaligned, off); + /* Loads and sign extending loads, also into the base */ + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + load_offsets(&m, K_LDX, size, rb, rd, off, nr); + if (size != BPF_DW) + load_offsets(&m, K_LDSX, size, rb, rd, off, nr); + } + /* Stores of a register and of an immediate */ + for (k = 1; k < NR_PAIRS; k++) + store_offsets(&m, size, rb, pair_reg(b, k), 0, off, nr); + for (k = 0; k < ARRAY_SIZE(imms); k++) + store_offsets(&m, size, rb, -1, imms[k], off, nr); + } + } +out: + free(m.mem); + free(m.exp); + free(m.res); + close(m.map_fd); +} + +static void test_movsx(void) +{ + static const struct { + bool is64; + int off; + } ops[] = { + { true, 8 }, { true, 16 }, { true, 32 }, { false, 8 }, { false, 16 }, + }; + struct prog p; + char desc[64]; + int s, k, o, j, fd; + + for (s = 0; s < ARRAY_SIZE(pat_regs); s++) { + int rs = pat_regs[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(s, k); + + for (o = 0; o < ARRAY_SIZE(ops); o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_6, IN0)); + if (ops[o].is64) + EMIT(&p, BPF_MOVSX64_REG(rd, rs, ops[o].off)); + else + EMIT(&p, BPF_MOVSX32_REG(rd, rs, ops[o].off)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "movsx%d off=%d dst=r%d src=r%d", + ops[o].is64 ? 64 : 32, ops[o].off, rd, rs); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = inputs[j]; + + if (ops[o].off == 8) + v = (__s8)v; + else if (ops[o].off == 16) + v = (__s16)v; + else + v = (__s32)v; + prog_run(fd, inputs[j], 0, + ops[o].is64 ? (__u64)v : (__u32)v, desc); + } + close(fd); + } + } + } +} + +/* Branch on dst and imm, with 1 in res when taken and 0 otherwise */ +static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm, int off) +{ + int rd = pat_regs[d], rt = pair_reg(d, 1); + + prog_start(p); + EMIT(p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(p, BPF_MOV64_IMM(rt, 1)); + if (is64) + EMIT(p, BPF_JMP_IMM(op, rd, imm, off)); + else + EMIT(p, BPF_JMP32_IMM(op, rd, imm, off)); + EMIT(p, BPF_MOV64_IMM(rt, 0)); + prog_end(p, rt); +} + +static void test_jset_imm(void) +{ + static const __s32 masks[] = { + 0x1, 0x40, 0x80, 0xff, 0x100, 0x7fff, 0x7fffffff, -1, -0x7fffffff - 1, + }; + struct prog p; + char desc[64]; + int d, m, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (m = 0; m < ARRAY_SIZE(masks); m++) { + jmp_prog(&p, is64, BPF_JSET, d, masks[m], 1); + snprintf(desc, sizeof(desc), "jset%d dst=r%d mask=0x%x", + is64 ? 64 : 32, rd, masks[m]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = inputs[j]; + bool taken; + + if (is64) + taken = v & (__u64)(__s64)masks[m]; + else + taken = (__u32)v & (__u32)masks[m]; + prog_run(fd, v, 0, taken, desc); + } + close(fd); + } + if (is64) + break; + } + } +} + +static bool cmp_taken(int op, bool is64, __u64 v, __u64 c) +{ + __s64 sv = is64 ? (__s64)v : (__s32)v, sc = is64 ? (__s64)c : (__s32)c; + + switch (op) { + case BPF_JEQ: + return v == c; + case BPF_JNE: + return v != c; + case BPF_JGT: + return v > c; + case BPF_JGE: + return v >= c; + case BPF_JLT: + return v < c; + case BPF_JLE: + return v <= c; + case BPF_JSGT: + return sv > sc; + case BPF_JSGE: + return sv >= sc; + case BPF_JSLT: + return sv < sc; + default: + return sv <= sc; + } +} + +static void test_cmp_imm(void) +{ + static const __s32 imms[] = { + 1, -1, 0x7f, 0x80, -0x80, -0x81, 0x1000, 0x7fffffff, + }; + static const int ops[] = { + BPF_JEQ, BPF_JNE, BPF_JGT, BPF_JGE, BPF_JLT, BPF_JLE, + BPF_JSGT, BPF_JSGE, BPF_JSLT, BPF_JSLE, + }; + struct prog p; + char desc[64]; + int d, o, m, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + for (m = 0; m < ARRAY_SIZE(imms); m++) { + jmp_prog(&p, is64, ops[o], d, imms[m], 1); + snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d imm=%d", + is64 ? 64 : 32, ops[o], rd, imms[m]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = is64 ? inputs[j] : (__u32)inputs[j]; + __u64 c = is64 ? (__u64)(__s64)imms[m] : + (__u32)imms[m]; + __u64 taken = cmp_taken(ops[o], is64, v, c); + + prog_run(fd, inputs[j], 0, taken, desc); + } + close(fd); + if (m) + continue; + /* A jump over nothing lands on the same path either way */ + jmp_prog(&p, is64, ops[o], d, imms[m], 0); + snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d off=0", + is64 ? 64 : 32, ops[o], rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, 0, desc); + close(fd); + } + } + if (is64) + break; + } + } +} + +static __u64 alu_model(int op, __u64 v, __u64 c) +{ + switch (op) { + case BPF_ADD: + return v + c; + case BPF_SUB: + return v - c; + case BPF_AND: + return v & c; + case BPF_OR: + return v | c; + case BPF_XOR: + return v ^ c; + default: + return v * c; + } +} + +static void test_alu_consts(void) +{ + static const struct { + int op; + __s32 imm; + } ops[] = { + { BPF_ADD, 1 }, { BPF_ADD, -1 }, { BPF_SUB, 1 }, { BPF_SUB, -1 }, + { BPF_ADD, 128 }, { BPF_SUB, 128 }, { BPF_ADD, -128 }, + { BPF_AND, 0xff }, { BPF_AND, 0xffff }, { BPF_AND, 0x7f }, + { BPF_AND, 0x3ff }, { BPF_AND, 0x7fffffff }, + { BPF_OR, 0xff }, { BPF_XOR, -1 }, { BPF_XOR, 0x80 }, + { BPF_MUL, 2 }, { BPF_MUL, 3 }, { BPF_MUL, 4 }, { BPF_MUL, 5 }, + { BPF_MUL, 7 }, { BPF_MUL, 8 }, { BPF_MUL, 9 }, { BPF_MUL, 16 }, + { BPF_MUL, 0x40000000 }, { BPF_MUL, -3 }, { BPF_MUL, -0x7fffffff - 1 }, + { BPF_MUL, 0 }, { BPF_MUL, 1 }, { BPF_MUL, -1 }, + }; + struct prog p; + char desc[64]; + int d, o, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + if (is64) + EMIT(&p, BPF_ALU64_IMM(ops[o].op, rd, ops[o].imm)); + else + EMIT(&p, BPF_ALU32_IMM(ops[o].op, rd, ops[o].imm)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "alu%d op=0x%x dst=r%d imm=%d", + is64 ? 64 : 32, ops[o].op, rd, ops[o].imm); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = is64 ? inputs[j] : (__u32)inputs[j]; + __u64 c = is64 ? (__u64)(__s64)ops[o].imm : + (__u32)ops[o].imm; + __u64 r = alu_model(ops[o].op, v, c); + + prog_run(fd, inputs[j], 0, is64 ? r : (__u32)r, desc); + } + close(fd); + } + if (is64) + break; + } + } +} + +static void test_bswap(void) +{ + static const int widths[] = { 16, 32, 64 }; + struct prog p; + char desc[64]; + int d, w, o, j, fd; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (w = 0; w < ARRAY_SIZE(widths); w++) { + int bits = widths[w]; + + /* To big endian, to little endian, and the unconditional swap */ + for (o = 0; o < 3; o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + if (o == 0) + EMIT(&p, BPF_ENDIAN(BPF_TO_BE, rd, bits)); + else if (o == 1) + EMIT(&p, BPF_ENDIAN(BPF_TO_LE, rd, bits)); + else + EMIT(&p, BPF_RAW_INSN(BPF_ALU64 | BPF_END | BPF_TO_LE, + rd, 0, 0, bits)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "%s%d dst=r%d", + o == 0 ? "be" : o == 1 ? "le" : "bswap", bits, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = inputs[j], r; + + if (bits == 16) + r = o == 0 ? htobe16(v) : o == 1 ? htole16(v) : + __builtin_bswap16(v); + else if (bits == 32) + r = o == 0 ? htobe32(v) : o == 1 ? htole32(v) : + __builtin_bswap32(v); + else + r = o == 0 ? htobe64(v) : o == 1 ? htole64(v) : + __builtin_bswap64(v); + prog_run(fd, v, 0, r, desc); + } + close(fd); + } + } + } +} + +static void test_probe_mem(void) +{ + static const struct { + int size; + bool sx; + } loads[] = { + { BPF_B, false }, { BPF_H, false }, { BPF_W, false }, { BPF_DW, false }, + { BPF_B, true }, { BPF_H, true }, { BPF_W, true }, + }; + int kfunc_id, type_id, b, k, l, j, fd; + struct bpf_insn call; + struct btf *btf; + struct prog p; + char desc[64]; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "vmlinux btf")) + return; + kfunc_id = btf__find_by_name_kind(btf, "bpf_rdonly_cast", BTF_KIND_FUNC); + type_id = btf__find_by_name_kind(btf, "timespec64", BTF_KIND_STRUCT); + btf__free(btf); + if (!ASSERT_GT(kfunc_id, 0, "bpf_rdonly_cast id") || + !ASSERT_GT(type_id, 0, "timespec64 id")) + return; + call = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, kfunc_id); + + /* + * The kernel copy of the ctx through an untrusted pointer in every + * register, in every size, zero and sign extending over all ones + */ + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + for (l = 0; l < ARRAY_SIZE(loads); l++) { + int size = loads[l].size, mode = loads[l].sx ? BPF_MEMSX : BPF_MEM; + + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, call); + if (rb != BPF_REG_0) + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0)); + if (rd != rb) + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + EMIT(&p, BPF_RAW_INSN(BPF_LDX | mode | size, rd, rb, 0, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), + "probe_mem%s size=%d base=r%d dst=r%d", + loads[l].sx ? "sx" : "", size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = head_bytes(inputs[j], size); + + if (loads[l].sx) + v = sign_extend(v, size); + prog_run(fd, inputs[j], 0, v, desc); + } + close(fd); + } + } + } + + /* NULL and a user address are not touched and read as zero */ + for (j = 0; j < 2; j++) { + __u64 addr = j ? 0x00007ffffffff000ULL : 0; + + prog_start(&p); + EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, 0, 0, (__u32)addr)); + EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, addr >> 32)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, call); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0)); + prog_end(&p, BPF_REG_3); + snprintf(desc, sizeof(desc), "probe_mem addr=0x%llx", addr); + fd = prog_load(&p, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, 0, desc); + close(fd); + } +} + +/* + * Load-acquire and store-release of every size with every register as + * base, on a stack slot holding in0: the loads zero extend like the plain + * loads and the stores replace the first bytes of the slot like the plain + * stores. + */ +static void test_acq_rel(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + struct prog p; + char desc[64]; + int b, k, s, j, fd; + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + /* Load-acquire from the slot, also into the base */ + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, BPF_LOAD_ACQ, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "load_acq size=%d base=r%d dst=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, + head_bytes(inputs[j], size), desc); + close(fd); + + if (rd == rb) + continue; + /* Store-release of in1 over the slot holding in0 */ + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, BPF_STORE_REL, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "store_rel size=%d base=r%d src=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], in1, size), desc); + } + close(fd); + } + } + } +} + +/* + * A jump with the maximal offset over instructions which expand to several + * native ones, so that the native offset outgrows the field of a short + * branch and takes the JITs through their branch conversion. The region + * must not grow in the verifier, or the jump could not reach over it any + * more: the divisions are by constants, as a division by a register gets + * a zero check in front, and the region sticks to one width, as a 32-bit + * result read by a 64-bit instruction gets a zero extension on the JITs + * which need one. The region is the not taken path and ends with a result + * different from the taken one, so a jump landing inside it fails the run. + */ +static void max_jmp(bool alu32, bool taken) +{ + static const int ops[] = { + BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, + }; + struct { __u64 r0, r2; } ctx = { taken, 0x0123456789abcdefULL }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + int len = INT16_MAX + 5, i = 0, fd; + struct bpf_insn *insns; + char desc[64]; + + snprintf(desc, sizeof(desc), "max jump alu%d %s", alu32 ? 32 : 64, + taken ? "taken" : "not taken"); + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0); + insns[i++] = BPF_MOV64_REG(BPF_REG_3, BPF_REG_2); + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, INT16_MAX); + while (i < len - 2) { + int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; + + if (op == BPF_DIV || op == BPF_MOD) + insns[i++] = alu32 ? BPF_ALU32_IMM(op, BPF_REG_3, 7) : + BPF_ALU64_IMM(op, BPF_REG_3, 7); + else + insns[i++] = alu32 ? BPF_ALU32_REG(op, BPF_REG_3, BPF_REG_2) : + BPF_ALU64_REG(op, BPF_REG_3, BPF_REG_2); + } + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 2); + insns[i++] = BPF_EXIT_INSN(); + ASSERT_EQ(i, len, "insn count"); + + fd = load_prog(insns, i, desc); + if (fd >= 0) { + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(opts.retval, taken ? 1 : 2, desc); + close(fd); + } + free(insns); +} + +static void test_max_jmp(void) +{ + max_jmp(false, true); + max_jmp(false, false); + max_jmp(true, true); + max_jmp(true, false); +} + +/* + * Fillers of two and three bytes of native code on x86-64: a 32-bit and a + * 64-bit move between two low registers, which give a region every length + * from two bytes on and which constant blinding leaves alone. + */ +static int fill_bytes(struct bpf_insn *insns, int n, int bytes) +{ + if (bytes & 1) { + insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3); + bytes -= 3; + } + for (; bytes > 0; bytes -= 2) + insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3); + return n; +} + +static void prog_result(struct bpf_insn *insns, int *n) +{ + insns[(*n)++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10); + insns[(*n)++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES); + insns[(*n)++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[(*n)++] = BPF_EXIT_INSN(); +} + +/* A region of x86-64 bytes through fill_bytes(), or of so many 64 and 32-bit moves */ +struct fill { + int bytes, wide, narrow; +}; + +static int fill_moves(struct bpf_insn *insns, int n, int wide, int narrow) +{ + while (wide-- > 0) + insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3); + while (narrow-- > 0) + insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3); + return n; +} + +static int fill_region(struct bpf_insn *insns, int n, const struct fill *f) +{ + if (f->wide || f->narrow) + return fill_moves(insns, n, f->wide, f->narrow); + return fill_bytes(insns, n, f->bytes); +} + +static int fill_insns(const struct fill *f) +{ + return f->wide + f->narrow + f->bytes / 2 + 1; +} + +/* + * Jumps whose lengths depend on each other: a conditional jump forward + * over a region, a jump back from its end to its start or to the jump in + * front of it, and an unconditional jump out of it. The region takes every + * length around the short branch limit, in front of the exit and behind + * it, so the three jumps cross the limit at different lengths and in both + * directions. A chain of nop jumps in it, gone one per pass, shifts the + * phase in which the cycle meets the padding passes, as settling jumps + * elsewhere do. This is generic to any JIT with short and long branch + * forms: the ranges of the running one are probed, see find_ranges(), and + * the cycle swept across each of them. r0 goes in with 0 to 3 and picks + * one of the four paths through the jumps. + */ +static void jump_cycle(bool back_cond, bool to_start, int inner, const struct fill *front, + const struct fill *back, const char *desc) +{ + struct bpf_insn *insns; + int n = 0, disp, head, chain, to_back[2], out, i, fd; + + insns = malloc((32 + 2 * inner + fill_insns(front) + fill_insns(back)) * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, desc)) + return; + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1); + insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + /* The dispatch into the chain, see nop_chain() */ + disp = n; + n += inner; + head = n++; + n = fill_region(insns, n, front); + chain = n; + for (i = 0; i < inner; i++) { + insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1); + insns[n++] = BPF_JMP_IMM(BPF_JA, 0, 0, inner - 1 - i); + } + to_back[0] = n++; + to_back[1] = n++; + out = n++; + insns[to_back[0]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, n - to_back[0] - 1); + insns[to_back[1]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 3, n - to_back[1] - 1); + insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1); + n = fill_region(insns, n, back); + /* Taken with 2 and not with 4, so the verifier keeps the condition */ + if (back_cond) + insns[n] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 4, head + to_start - n - 1); + else + insns[n] = BPF_JMP_IMM(BPF_JA, 0, 0, head + to_start - n - 1); + n++; + insns[head] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, n - head - 1); + insns[out] = BPF_JMP_IMM(BPF_JA, 0, 0, n - out - 1); + prog_result(insns, &n); + + fd = load_prog(insns, n, desc); + free(insns); + if (fd < 0) + return; + prog_run(fd, 0, 0, 10, desc); + prog_run(fd, 1, 0, 12, desc); + prog_run(fd, 2, 0, 12, desc); + prog_run(fd, 3, 0, 14, desc); + close(fd); +} + +/* + * A region which ends in a chain of jumps to its end: the last one jumps + * to the next instruction, which the JIT drops as a nop, whereupon the one + * before it becomes one, so the chain goes away one jump per pass and the + * jump over the region shrinks late, in the passes which pad the image. + * The filler in front of the chain puts that jump around the short branch + * limit, as a conditional and as an unconditional one, over a chain of ja + * and one of gotol. Each jump of the chain is the target of a conditional + * jump in front of the region on a value the verifier cannot see, as the + * chain is dead behind its first jump otherwise. + */ +static void nop_chain(bool over_cond, bool ja32, int len, int fill) +{ + struct bpf_insn insns[256]; + int n = 0, disp, over, chain, i, fd; + char desc[64]; + + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1); + insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + /* The dispatch into the chain, filled in once the chain is placed */ + disp = n; + n += len; + if (!over_cond) + insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, 1); + over = n++; + n = fill_bytes(insns, n, fill); + chain = n; + for (i = 0; i < len; i++) { + insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1); + insns[n++] = ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, len - 1 - i) : + BPF_JMP_IMM(BPF_JA, 0, 0, len - 1 - i); + } + insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1); + if (over_cond) + insns[over] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 2, n - over - 1); + else + insns[over] = BPF_JMP_IMM(BPF_JA, 0, 0, n - over - 1); + prog_result(insns, &n); + + snprintf(desc, sizeof(desc), "nop chain of %s under %s len=%d fill=%d", + ja32 ? "gotol" : "ja", over_cond ? "jcc" : "ja", len, fill); + fd = load_prog(insns, n, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, 10, desc); + prog_run(fd, 1, 0, 11, desc); + prog_run(fd, 2, 0, 13, desc); + prog_run(fd, 3, 0, 13, desc); + close(fd); +} + +/* The native length of a program's image, 0 without a JIT */ +static __u32 jited_len(int fd) +{ + struct bpf_prog_info info = {}; + __u32 len = sizeof(info); + + if (!ASSERT_OK(bpf_prog_get_info_by_fd(fd, &info, &len), "prog info")) + return 0; + return info.jited_prog_len; +} + +/* A conditional jump over the moves, for the length of its image */ +static int probe_len(int wide, int narrow, __u32 *len) +{ + struct bpf_insn *insns; + int n = 0, fd; + + insns = malloc((wide + narrow + 8) * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "probe insns")) + return -1; + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, wide + narrow); + n = fill_moves(insns, n, wide, narrow); + insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[n++] = BPF_EXIT_INSN(); + fd = load_prog(insns, n, "branch range probe"); + free(insns); + if (fd < 0) + return -1; + *len = jited_len(fd); + close(fd); + return 0; +} + +/* + * The short branch ranges of the running JIT, as counts of 64-bit moves a + * conditional jump reaches over, and the native lengths of the two moves. + * The image of the probe grows by the moves alone until the jump does not + * reach over them any more and takes a longer form, which adds the + * difference of the forms: find the first count that does so in every + * power of two of moves up to the 16-bit offset of the jump, as a JIT may + * have several forms. Without a JIT or with fixed length branches there is + * none. + */ +static int find_ranges(int *ranges, int max, int *wide_bytes, int *narrow_bytes) +{ + __u32 base, len; + int excess, plateau = 0, c, lo, hi, mid, k = 0; + + if (probe_len(8, 0, &base) || !base) + return 0; + if (probe_len(24, 0, &len)) + return 0; + *wide_bytes = (len - base) / 16; + if (probe_len(8, 16, &len)) + return 0; + *narrow_bytes = (len - base) / 16; + if (!*wide_bytes || !*narrow_bytes) + return 0; + for (c = 32; k < max && c <= 16384; c *= 2) { + if (probe_len(c, 0, &len)) + return k; + excess = len - base - (c - 8) * *wide_bytes; + if (excess <= plateau) + continue; + lo = c / 2; + hi = c; + while (hi - lo > 1) { + mid = (lo + hi) / 2; + if (probe_len(mid, 0, &len)) + return k; + if (len - base - (mid - 8) * *wide_bytes > plateau) + hi = mid; + else + lo = mid; + } + ranges[k++] = hi; + plateau = excess; + } + return k; +} + +static int gcd(int a, int b) +{ + while (b) { + int t = a % b; + + a = b; + b = t; + } + return a; +} + +/* The jump cycle with the region in front of the exit and then behind it */ +static void cycle_bytes(int kind, int inner, int bytes) +{ + struct fill front = { .bytes = bytes }, back = {}; + char desc[96]; + + snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d front=%d", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes); + jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc); + front.bytes = 0; + back.bytes = bytes; + snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d back=%d", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes); + jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc); +} + +/* The jump cycle with a region of so many native bytes of moves */ +static void cycle_moves(int kind, int range, int bytes, int wide, int narrow) +{ + struct fill front = {}, back = {}; + char desc[96]; + int b; + + for (b = 0; b * narrow <= bytes; b++) + if ((bytes - b * narrow) % wide == 0) + break; + if (b * narrow > bytes) + return; + front.wide = (bytes - b * narrow) / wide; + front.narrow = b; + snprintf(desc, sizeof(desc), "jump cycle %s back to %s range %d at %d bytes", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", range, bytes); + jump_cycle(kind & 1, kind & 2, 0, &front, &back, desc); +} + +static void test_convergence(void) +{ + /* The chain gone, the jump over it is at and below the limit */ + static const int fills[] = { 100, 110, 116, 120 }; + /* The last jump of a chain is a nop the verifier already removes */ + static const int inners[] = { 0, 2, 3, 4, 5 }; + int ranges[4], wide, narrow, g, nr, bytes, lo, hi, c, f, i, l, k; + + /* The x86-64 imm8 range, byte by byte */ + for (c = 0; c < 4; c++) + for (i = 0; i < ARRAY_SIZE(inners); i++) + for (f = 100; f <= 132; f++) + cycle_bytes(c, inners[i], f); + for (c = 0; c < 4; c++) + for (f = 0; f < ARRAY_SIZE(fills); f++) + for (l = 2; l <= 26; l++) + nop_chain(c & 1, c & 2, l, fills[f]); + + /* + * The short branch ranges of this JIT, which the fillers of 64 and + * 32-bit moves approach at the granularity their lengths allow + */ + nr = find_ranges(ranges, ARRAY_SIZE(ranges), &wide, &narrow); + for (k = 0; k < nr; k++) { + printf("short branch range at %d bytes, moves of %d and %d bytes\n", + ranges[k] * wide, wide, narrow); + g = gcd(wide, narrow); + lo = ranges[k] * wide - 128; + hi = ranges[k] * wide + 32; + for (c = 0; c < 4; c++) + for (bytes = lo; bytes <= hi; bytes += g) + cycle_moves(c, ranges[k] * wide, bytes, wide, narrow); + } +} + +/* + * Calls into a subprogram with all five arguments, which clobbers a subset + * of r6-r9 and uses the deepest stack slot the budget leaves, directly or + * through another one clobbering the rest, so the JIT has to save and + * restore what it uses. + */ +/* Values to keep in r6-r9 across calls */ +static const __u64 kept[] = { + 0x1111111111111111ULL, 0x2222222222222222ULL, + 0x3333333333333333ULL, 0x4444444444444444ULL, +}; + +static void test_calls(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + __u64 expected = 1 + 2 + 3 + 4 + 5; + struct bpf_insn insns[64]; + struct pat_ctx ctx; + char desc[64]; + int nested, mask, n, call, r, fd, leaf; + + /* The frames above the leaf take up to 32 bytes each of the budget */ + leaf = (is_large_stack_supported() ? 2048 : 512) - 2 * 32; + for (r = 0; r < 4; r++) + expected += kept[r]; + opts.ctx_in = &ctx; + opts.ctx_size_in = sizeof(ctx); + for (nested = 0; nested < 2; nested++) { + for (mask = 0; mask < 16; mask++) { + n = 0; + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -8); + for (r = 0; r < 4; r++) { + insns[n++] = BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_6 + r, + 0, 0, (__u32)kept[r]); + insns[n++] = BPF_RAW_INSN(0, 0, 0, 0, kept[r] >> 32); + } + for (r = 1; r <= 5; r++) + insns[n++] = BPF_MOV64_IMM(r, r); + call = n++; + for (r = 6; r <= 9; r++) + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -8); + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES); + insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[n++] = BPF_EXIT_INSN(); + insns[call] = BPF_CALL_REL(n - call - 1); + if (nested) { + /* Clobber the subset, pass the arguments on */ + for (r = 0; r < 4; r++) + if (mask & (1 << r)) + insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0); + call = n++; + insns[n++] = BPF_EXIT_INSN(); + insns[call] = BPF_CALL_REL(n - call - 1); + } + /* Clobber the rest, sum the arguments through the deepest slot */ + for (r = 0; r < 4; r++) + if ((nested ? ~mask : mask) & (1 << r)) + insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0); + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -leaf); + insns[n++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); + for (r = 3; r <= 5; r++) + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -leaf); + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); + insns[n++] = BPF_EXIT_INSN(); + + snprintf(desc, sizeof(desc), "call%s clobbering 0x%x", + nested ? " nested" : "", mask); + fd = load_prog(insns, n, desc); + if (fd < 0) + return; + memset(&ctx, 0, sizeof(ctx)); + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(ctx.res, expected, desc); + close(fd); + } + } +} + +/* + * Helper calls with up to five arguments and a kfunc call: a dynptr over a + * map value, a read from it into the stack and its size, with r6-r9 live + * across the calls. The returns, the bytes read, the size and r6-r9 add up. + */ +static void test_helper_calls(void) +{ + static const int offs[] = { 0, 8, 1000, 4088 }; + struct off_prog p; + struct btf *btf; + __u8 mem[4096]; + __u64 expected; + __u32 key = 0; + char desc[64]; + int map_fd, kfunc_id, i, r, fd; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "vmlinux btf")) + return; + kfunc_id = btf__find_by_name_kind(btf, "bpf_dynptr_size", BTF_KIND_FUNC); + btf__free(btf); + if (!ASSERT_GT(kfunc_id, 0, "bpf_dynptr_size id")) + return; + for (i = 0; i < sizeof(mem); i++) + mem[i] = i * 7 + 3; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "helper_mem", sizeof(key), sizeof(mem), 1, + NULL); + if (!ASSERT_GE(map_fd, 0, "helper_mem map")) + return; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, mem, 0), "helper_mem update")) + goto out; + + for (i = 0; i < ARRAY_SIZE(offs); i++) { + p.cnt = 0; + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -64)); + for (r = 0; r < 4; r++) + off_prog_ld_imm64(&p, BPF_REG_6 + r, kept[r]); + off_prog_lookup(&p, map_fd, key); + /* The dynptr at -80 over the value, the returns summed at -96 */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_0)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, sizeof(mem))); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_3, 0)); + EMIT(&p, BPF_MOV64_REG(BPF_REG_4, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -80)); + EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_from_mem)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96)); + /* Eight bytes from the offset into -88 */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -88)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, 8)); + EMIT(&p, BPF_MOV64_REG(BPF_REG_3, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -80)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_4, offs[i])); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_5, 0)); + EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_read)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96)); + /* The size through the kfunc */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -80)); + EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, + kfunc_id)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -88)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + for (r = BPF_REG_6; r <= BPF_REG_9; r++) + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -64)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES)); + snprintf(desc, sizeof(desc), "helper calls off=%d", offs[i]); + fd = off_prog_load(&p, desc, false); + if (fd < 0) + goto out; + expected = sizeof(mem) + load_bytes(mem + offs[i], BPF_DW); + for (r = 0; r < 4; r++) + expected += kept[r]; + prog_run(fd, 0, 0, expected, desc); + close(fd); + } +out: + close(map_fd); +} + +void test_jit_insns(void) +{ + RUN_TESTS(jit_insns_arena); + if (test__start_subtest("alu_imm")) + test_alu_imm(); + if (test__start_subtest("alu_reg")) + test_alu_reg(); + if (test__start_subtest("alu_imm_regs")) + test_alu_imm_regs(); + if (test__start_subtest("div_const")) + test_div_const(); + if (test__start_subtest("alu_reg_pairs")) + test_alu_reg_pairs(); + if (test__start_subtest("alu_edges")) + test_alu_edges(); + if (test__start_subtest("shifts")) + test_shifts(); + if (test__start_subtest("shift_counts")) + test_shift_counts(); + if (test__start_subtest("atomics")) + test_atomics(); + if (test__start_subtest("atomic_reg_pairs")) + test_atomic_reg_pairs(); + if (test__start_subtest("xadd_chain")) + test_xadd_chain(); + if (test__start_subtest("ld_imm64")) + test_ld_imm64(); + if (test__start_subtest("jmp_imm")) + test_jmp_imm(); + if (test__start_subtest("jmp_reg")) + test_jmp_reg(); + if (test__start_subtest("jmp_reg_pairs")) + test_jmp_reg_pairs(); + if (test__start_subtest("jmp_edges")) + test_jmp_edges(); + if (test__start_subtest("staggered_jumps")) + test_staggered_jumps(); + if (test__start_subtest("max_jmp")) + test_max_jmp(); + if (test__start_subtest("convergence")) + test_convergence(); + if (test__start_subtest("calls")) + test_calls(); + if (test__start_subtest("helper_calls")) + test_helper_calls(); + if (test__start_subtest("ldst_zero_off")) + test_ldst_zero_off(); + if (test__start_subtest("ldst_offsets")) + test_ldst_offsets(); + if (test__start_subtest("acq_rel")) + test_acq_rel(); + if (test__start_subtest("movsx")) + test_movsx(); + if (test__start_subtest("jset_imm")) + test_jset_imm(); + if (test__start_subtest("cmp_imm")) + test_cmp_imm(); + if (test__start_subtest("alu_consts")) + test_alu_consts(); + if (test__start_subtest("bswap")) + test_bswap(); + if (test__start_subtest("probe_mem")) + test_probe_mem(); +} + +/* The families with an immediate once more with the constants blinded */ +void serial_test_jit_insns_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + if (test__start_subtest("alu_imm")) + test_alu_imm(); + if (test__start_subtest("alu_imm_regs")) + test_alu_imm_regs(); + if (test__start_subtest("alu_edges")) + test_alu_edges(); + if (test__start_subtest("div_const")) + test_div_const(); + if (test__start_subtest("shifts")) + test_shifts(); + if (test__start_subtest("xadd_chain")) + test_xadd_chain(); + if (test__start_subtest("ld_imm64")) + test_ld_imm64(); + if (test__start_subtest("jmp_imm")) + test_jmp_imm(); + if (test__start_subtest("jmp_edges")) + test_jmp_edges(); + if (test__start_subtest("ldst_zero_off")) + test_ldst_zero_off(); + if (test__start_subtest("ldst_offsets")) + test_ldst_offsets(); + if (test__start_subtest("jset_imm")) + test_jset_imm(); + if (test__start_subtest("cmp_imm")) + test_cmp_imm(); + if (test__start_subtest("alu_consts")) + test_alu_consts(); + sysctl_set_or_fail(harden, NULL, old); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/kasan.c b/tools/testing/selftests/bpf/prog_tests/kasan.c new file mode 100644 index 0000000..9911b96 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kasan.c
@@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +/* + * Tests validating that KASAN reports are properly instrumented and + * generated on a wide variety of instructions. The running kernel needs + * kasan_multi_shot to run multiple kasan-generating subtests at once + */ +#include <bpf/bpf.h> +#include <errno.h> +#include <fcntl.h> +#include <linux/if_ether.h> +#include <unistd.h> +#include <test_progs.h> +#include <unpriv_helpers.h> +#include "sysctl_helpers.h" +#include "kasan.skel.h" +#include "kasan_harden.skel.h" + +#define SUBTEST_NAME_MAX_LEN 128 +#define PROG_NAME_MAX_LEN 128 + +#define MAX_LOG_SIZE (8 * 1024) +#define READ_CHUNK_SIZE 256 + +#define KASAN_PATTERN_SLAB_UAF "BUG: KASAN: slab-use-after-free " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_SLAB_OOB "BUG: KASAN: slab-out-of-bounds " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_REPORT "%s of size %d at addr" + +static char klog_buffer[MAX_LOG_SIZE]; +static char record[MAX_LOG_SIZE]; + +struct test_spec { + char *prog_type; + bool is_write; + bool only_32_or_64; + bool needs_load_acq_store_rel; + bool needs_st; + bool skip_multi_size_testing; + bool skip_on_stack_testing; + int run_size; + bool expect_no_report; + bool rnd_hi32; + bool is_oob; +}; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct test_ctx { + __u8 prog_tag[BPF_TAG_SIZE]; + struct bpf_object *obj; + int *access_size; + bool skip_load_acq_store_rel; + bool skip_st_tests; + struct bpf_program *prog; + char prog_name[SUBTEST_NAME_MAX_LEN]; + int klog_fd; +}; + +static int open_kernel_logs(void) +{ + int fd; + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + + return fd; +} + +static void skip_kernel_logs(int fd) +{ + lseek(fd, 0, SEEK_END); +} + +static int read_kernel_logs(int fd, char *buf, size_t max_len) +{ + size_t total = 0; + ssize_t n; + + buf[0] = '\0'; + while (1) { + char *msg, *eol; + size_t len; + + n = read(fd, record, sizeof(record) - 1); + if (n == 0) + break; + + if (n < 0) { + if (errno == EAGAIN) + break; + return n; + } + record[n] = '\0'; + + /* + * Each kmsg record starts with some metadata, separated + * from the actual content by a semi-colon + */ + msg = strchr(record, ';'); + if (!msg) + continue; + msg++; + eol = strchr(msg, '\n'); + if (eol) + *eol = '\0'; + + len = strlen(msg); + if (total + len + 2 > max_len) + break; + memcpy(buf + total, msg, len); + total += len; + buf[total++] = '\n'; + buf[total] = '\0'; + } + + return total; +} + +static int check_kasan_report_in_kernel_logs(char *buf, struct test_ctx *ctx, + bool is_write, int size, + bool is_oob) +{ + char access_log[READ_CHUNK_SIZE]; + const char *pattern; + char *kasan_report_start; + int nsize; + + pattern = is_oob ? KASAN_PATTERN_SLAB_OOB : KASAN_PATTERN_SLAB_UAF; + nsize = snprintf(access_log, READ_CHUNK_SIZE, pattern, + ctx->prog_tag[0], ctx->prog_tag[1], ctx->prog_tag[2], + ctx->prog_tag[3], ctx->prog_tag[4], ctx->prog_tag[5], + ctx->prog_tag[6], ctx->prog_tag[7], ctx->prog_name); + if (!ASSERT_GE(nsize, 0, "format kasan access header line")) + return nsize; + /* + * Searched kasan report is valid if + * - it contains the expected kasan pattern + * - the description of the faulty access is found somewhere + * after the header (not necessarily on the very next line, + * because other kernel messages may interleave) + * - faulty access properties match the tested type and size + */ + kasan_report_start = strstr(buf, access_log); + + if (!kasan_report_start) + return 1; + + nsize = snprintf(access_log, READ_CHUNK_SIZE, KASAN_PATTERN_REPORT, + is_write ? "Write" : "Read", size); + if (!ASSERT_GE(nsize, 0, "format kasan access report line")) + return nsize; + + if (!strstr(kasan_report_start, access_log)) + return 1; + + return 0; +} + +static void exec_subtest(struct test_ctx *ctx, struct test_spec *test, + int access_size, bool on_stack) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_prog_info info; + uint8_t buf[ETH_HLEN] = {0}; + int ret, prog_fd; + __u32 info_len; + + ctx->prog = bpf_object__find_program_by_name(ctx->obj, + ctx->prog_name); + if (!ASSERT_OK_PTR(ctx->prog, "find test prog")) + return; + + info_len = sizeof(info); + memset(&info, 0, info_len); + prog_fd = bpf_program__fd(ctx->prog); + if (!ASSERT_OK_FD(prog_fd, "get prog fd")) + return; + ret = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (!ASSERT_OK(ret, "fetch loaded program info")) + return; + memcpy(ctx->prog_tag, info.tag, BPF_TAG_SIZE); + + skip_kernel_logs(ctx->klog_fd); + + topts.sz = sizeof(struct bpf_test_run_opts); + topts.data_size_in = ETH_HLEN; + topts.data_in = buf; + if (ctx->access_size) + *ctx->access_size = access_size; + ret = bpf_prog_test_run_opts(bpf_program__fd(ctx->prog), + &topts); + if (!ASSERT_OK(ret, "run prog")) + return; + + ret = read_kernel_logs(ctx->klog_fd, klog_buffer, MAX_LOG_SIZE); + if (!ASSERT_GE(ret, 0, "read kernel logs")) + return; + + ret = check_kasan_report_in_kernel_logs(klog_buffer, ctx, + test->is_write, access_size, + test->is_oob); + if (on_stack || test->expect_no_report) + ASSERT_NEQ(ret, 0, "no report should be generated"); + else + ASSERT_OK(ret, "report should be generated"); +} + +static void run_subtest_with_size_and_location(struct test_ctx *ctx, + struct test_spec *test, + int access_size, + bool on_stack) +{ + char subtest_name[SUBTEST_NAME_MAX_LEN]; + + if (test->skip_multi_size_testing) { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s%s", + test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + } else { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s_%d_%s", + test->prog_type, access_size, + on_stack ? "on_stack" : "not_on_stack"); + } + + snprintf(ctx->prog_name, PROG_NAME_MAX_LEN, "%s%s", test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + + if (!test__start_subtest(subtest_name)) + return; + + if (test->needs_load_acq_store_rel && ctx->skip_load_acq_store_rel) { + test__skip(); + return; + } + + if (test->needs_st && ctx->skip_st_tests) { + test__skip(); + return; + } + + exec_subtest(ctx, test, access_size, on_stack); +} + +static void run_subtest_with_size(struct test_ctx *ctx, struct test_spec *test, + int size) +{ + run_subtest_with_size_and_location(ctx, test, size, false); + if (!test->skip_on_stack_testing) + run_subtest_with_size_and_location(ctx, test, size, true); +} + +static void run_subtest(struct test_ctx *ctx, struct test_spec *test) +{ + if (test->skip_multi_size_testing) { + run_subtest_with_size(ctx, test, test->run_size); + return; + } + + if (!test->only_32_or_64) { + run_subtest_with_size(ctx, test, 1); + run_subtest_with_size(ctx, test, 2); + } + run_subtest_with_size(ctx, test, 4); + run_subtest_with_size(ctx, test, 8); +} + +static void run_blinding_subtest(void) +{ + struct test_spec blinding_spec = { + .prog_type = "st_blinded", + .is_write = true, + }; + char bpf_jit_harden_orig[2]; + struct kasan_harden *skel; + struct test_ctx *ctx; + + if (!test__start_subtest("st_blinded")) + return; + + ctx = calloc(1, sizeof(*ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc blinding ctx")) + return; + ctx->klog_fd = -1; + + if (sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", + bpf_jit_harden_orig, "2")) + goto free_ctx; + + skel = kasan_harden__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load blinded prog")) + goto restore; + + if (skel->data->skip_st_tests) { + test__skip(); + goto destroy; + } + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + ctx->obj = skel->obj; + strncpy(ctx->prog_name, "st_blinded", PROG_NAME_MAX_LEN); + + exec_subtest(ctx, &blinding_spec, 1, false); + +destroy: + close(ctx->klog_fd); + kasan_harden__destroy(skel); +restore: + sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", NULL, + bpf_jit_harden_orig); +free_ctx: + free(ctx); +} + +static struct test_spec tests[] = { + { + .prog_type = "st", + .is_write = true, + .needs_st = true + }, + { + .prog_type = "stx", + .is_write = true + }, + { + .prog_type = "ldx", + .is_write = false + }, + { + .prog_type = "simple_atomic", + .is_write = true, + .only_32_or_64 = true + }, + { + .prog_type = "simple_atomic_fetch", + .is_write = true, + .skip_multi_size_testing = true, + .run_size = 8, + }, + { + .prog_type = "load_acquire", + .is_write = false, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "store_release", + .is_write = true, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "ldx_patched", + .is_write = false, + .skip_multi_size_testing = true, + .run_size = 4, + .rnd_hi32 = true + }, + { + .prog_type = "verifier_paths_stack_and_non_stack", + .is_write = true, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 1 + }, + { + .prog_type = "ldx_oob", + .is_write = false, + .skip_on_stack_testing = true, + .is_oob = true + }, + { + .prog_type = "ldx_self_alias_on_stack", + .is_write = false, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 8, + .expect_no_report = true + } +}; + +void serial_test_kasan(void) +{ + struct kasan_write_val val; + struct test_spec *test; + struct test_ctx *ctx; + struct kasan *skel; + __u32 key = 0; + int i, ret; + + ctx = calloc(1, sizeof(struct test_ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc test ctx")) + return; + + if (!is_jit_enabled() || !get_kasan_jit_enabled() || + !get_kasan_multi_shot_enabled()) { + test__skip(); + goto end; + } + + skel = kasan__open(); + if (!ASSERT_OK_PTR(skel, "open prog")) + goto end; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + char prog_name[SUBTEST_NAME_MAX_LEN]; + struct bpf_program *prog; + + if (!tests[i].rnd_hi32) + continue; + + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 on_stack prog")) + goto destroy; + bpf_program__add_flags(prog, BPF_F_TEST_RND_HI32); + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "not_on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 not_on_stack prog")) + goto destroy; + bpf_program__add_flags(prog, BPF_F_TEST_RND_HI32); + } + + if (!ASSERT_OK(kasan__load(skel), "load prog")) + goto destroy; + + ctx->obj = skel->obj; + ctx->access_size = &skel->bss->access_size; + ctx->skip_load_acq_store_rel = skel->data->skip_load_acq_store_rel_tests; + ctx->skip_st_tests = skel->data->skip_st_tests; + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + /* Fill map with recognizable values */ + ret = bpf_map__lookup_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "get map")) + goto close; + val.data_1 = 0xAA; + val.data_2 = 0xBBBB; + val.data_4 = 0xCCCCCCCC; + val.data_8 = 0xDDDDDDDDDDDDDDDD; + ret = bpf_map__update_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "set map")) + goto close; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test = &tests[i]; + run_subtest(ctx, test); + } + + /* + * Blinding subtest is handled differently as it needs the + * corresponding program to be loaded with bpf_jit_harden raised + */ + run_blinding_subtest(); + +close: + close(ctx->klog_fd); +destroy: + kasan__destroy(skel); +end: + free(ctx); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c index 97b00c7..c723c9f 100644 --- a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c +++ b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c
@@ -10,12 +10,32 @@ void test_kernel_flag(void) struct test_kernel_flag *lsm_skel; struct kfunc_call_test *skel = NULL; struct kfunc_call_test_lskel *lskel = NULL; + struct bpf_program *prog; + __u32 flags; int ret; - lsm_skel = test_kernel_flag__open_and_load(); + lsm_skel = test_kernel_flag__open(); if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel")) return; + prog = lsm_skel->progs.bpf; + flags = bpf_program__flags(prog); + ASSERT_EQ(flags & BPF_F_SLEEPABLE, + BPF_F_SLEEPABLE, "sleepable in program flags"); + + ret = bpf_program__add_flags(prog, BPF_F_ANY_ALIGNMENT); + ASSERT_OK(ret, "bpf_program__add_flags ret"); + ASSERT_EQ(bpf_program__flags(prog), flags | BPF_F_ANY_ALIGNMENT, + "bpf_program__add_flags value"); + + ret = bpf_program__clear_flags(prog, BPF_F_ANY_ALIGNMENT); + ASSERT_OK(ret, "bpf_program__clear_flags ret"); + ASSERT_EQ(bpf_program__flags(prog), flags, "bpf_program__clear_flags value"); + + ret = test_kernel_flag__load(lsm_skel); + if (!ASSERT_OK(ret, "test_kernel_flag__load")) + goto close_prog; + lsm_skel->bss->monitored_tid = sys_gettid(); ret = test_kernel_flag__attach(lsm_skel);
diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index 2b39cc1..0063e60 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
@@ -70,7 +70,7 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), - TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 type=scalar expected=ctx"), TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */
diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 2e0ddef..651123c 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
@@ -361,8 +361,7 @@ static void test_attach_api_fails(void) sl_skel->bss->user_ptr = sl_skel; - err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, - BPF_F_SLEEPABLE); + err = bpf_program__add_flags(sl_skel->progs.handle_kprobe_multi_sleepable, BPF_F_SLEEPABLE); if (!ASSERT_OK(err, "sleep_skel_set_flags")) goto cleanup;
diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_sendto.c b/tools/testing/selftests/bpf/prog_tests/ksock_sendto.c new file mode 100644 index 0000000..498f0e1 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock_sendto.c
@@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "test_progs.h" +#include "network_helpers.h" +#include "ksock_sendto.skel.h" + +struct sendto_test_case { + const char *name; + bool connected; + bool use_sendto_addr; +}; + +static bool recv_payload(int fd, const void *expected, size_t len) +{ + char buf[64]; + ssize_t n; + + n = recv(fd, buf, sizeof(buf), MSG_TRUNC); + if (!ASSERT_EQ(n, len, "recv length")) + return false; + return ASSERT_MEMEQ(buf, expected, len, "recv payload"); +} + +static void test_sendto_case(const struct sendto_test_case *test) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_sendto *skel; + int receivers[2] = { -1, -1 }; + int err, i, port, sendto_fd; + char buf[64]; + ssize_t n; + + skel = ksock_sendto__open_and_load(); + if (!ASSERT_OK_PTR(skel, "ksock_sendto open and load")) + return; + + /* Use separate receivers for the default peer and explicit destination. */ + for (i = 0; i < ARRAY_SIZE(receivers); i++) { + receivers[i] = start_server(AF_INET, SOCK_DGRAM, "127.0.0.1", 0, 1000); + if (!ASSERT_OK_FD(receivers[i], "receiver socket")) + goto out; + } + + port = get_socket_local_port(receivers[0]); + if (!ASSERT_GE(port, 0, "connected port")) + goto out; + skel->bss->connected_port = port; + port = get_socket_local_port(receivers[1]); + if (!ASSERT_GE(port, 0, "sendto port")) + goto out; + skel->bss->sendto_port = port; + skel->bss->connected = test->connected; + skel->bss->use_sendto_addr = test->use_sendto_addr; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_sendto), &opts); + if (!ASSERT_OK(err, "run test_sendto") || !ASSERT_OK(opts.retval, "test_sendto retval")) + goto out; + + /* Check sendto() outcome */ + if (!ASSERT_EQ(skel->data->sendto_ret, sizeof(skel->data->sendto_data), "sendto bytes")) + goto out; + sendto_fd = receivers[test->use_sendto_addr ? 1 : 0]; + if (!recv_payload(sendto_fd, skel->data->sendto_data, sizeof(skel->data->sendto_data))) + goto out; + + /* Check send() outcome */ + if (test->connected) { + if (!ASSERT_EQ(skel->data->send_ret, sizeof(skel->data->send_data), "send bytes")) + goto out; + if (!recv_payload(receivers[0], skel->data->send_data, + sizeof(skel->data->send_data))) + goto out; + } else if (!ASSERT_EQ(skel->data->send_ret, -EDESTADDRREQ, + "send after sendto remains unconnected")) { + goto out; + } + + /* Catch unexpected deliveries to either destination. */ + for (i = 0; i < ARRAY_SIZE(receivers); i++) { + n = recv(receivers[i], buf, sizeof(buf), MSG_DONTWAIT); + if (!ASSERT_EQ(n, -1, "no extra datagram") || + !ASSERT_EQ(errno, EAGAIN, "recv errno")) + goto out; + } + +out: + for (i = 0; i < ARRAY_SIZE(receivers); i++) { + if (receivers[i] >= 0) + close(receivers[i]); + } + ksock_sendto__destroy(skel); +} + +void test_ksock_sendto(void) +{ + static const struct sendto_test_case tests[] = { + { .name = "unconnected", .use_sendto_addr = true }, + { .name = "connected_null_addr", .connected = true }, + { .name = "connected_explicit_addr", .connected = true, .use_sendto_addr = true }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (test__start_subtest(tests[i].name)) + test_sendto_case(&tests[i]); + } +}
diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c index d6dc20b..12c2275 100644 --- a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c
@@ -4,6 +4,7 @@ #include <unistd.h> #include "test_progs.h" +#include "network_helpers.h" #include "ksock_wq.skel.h" #define CALLBACK_WAIT_RETRIES 1000 @@ -13,13 +14,22 @@ void test_ksock_wq(void) { LIBBPF_OPTS(bpf_test_run_opts, opts); struct ksock_wq *skel; + char recv_data[sizeof(skel->data->send_data)]; u32 callback_done; - int err, i; + int err, i, port, rfd; skel = ksock_wq__open_and_load(); if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) return; + rfd = start_server(AF_INET, SOCK_DGRAM, "127.0.0.1", 0, 1000); + if (!ASSERT_OK_FD(rfd, "receiver socket")) + goto out; + port = get_socket_local_port(rfd); + if (!ASSERT_GE(port, 0, "receiver port")) + goto out; + skel->bss->remote_port = port; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), &opts); if (!ASSERT_OK(err, "run ksock_wq_start")) @@ -28,18 +38,27 @@ void test_ksock_wq(void) goto out; for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { - if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE) == 2) break; usleep(CALLBACK_WAIT_US); } callback_done = __atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE); - if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + if (!ASSERT_EQ(callback_done, 2, "workqueue callbacks completed")) goto out; - ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, - "workqueue create rejected"); + ASSERT_EQ(skel->bss->create_ret, -EOPNOTSUPP, "workqueue create rejected"); + + if (!ASSERT_EQ(skel->bss->connect_send_ret, sizeof(recv_data), + "workqueue send bytes")) + goto out; + err = recv(rfd, recv_data, sizeof(recv_data), 0); + if (ASSERT_EQ(err, sizeof(recv_data), "recv len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); out: + if (rfd >= 0) + close(rfd); ksock_wq__destroy(skel); }
diff --git a/tools/testing/selftests/bpf/prog_tests/linked_externs.c b/tools/testing/selftests/bpf/prog_tests/linked_externs.c new file mode 100644 index 0000000..ff568f5 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/linked_externs.c
@@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <bpf/btf.h> +#include "linked_arena.skel.h" + +static void test_skel(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct linked_arena *skel; + int err; + + skel = linked_arena__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + if (skel->data->skip_tests) { + printf("%s:SKIP: no addr_space_cast support in the BPF compiler\n", __func__); + test__skip(); + goto cleanup; + } + + err = linked_arena__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + ASSERT_EQ(skel->arena->a_val, 1, "a_val_init"); + ASSERT_EQ(skel->arena->b_val, 2, "b_val_init"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.sum1), &opts); + ASSERT_OK(err, "sum1_run"); + ASSERT_EQ(opts.retval, 1 + 2, "sum1_retval"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.bump2), &opts); + ASSERT_OK(err, "bump2_run"); + ASSERT_EQ(opts.retval, 11 + 22, "bump2_retval"); + + ASSERT_EQ(skel->arena->a_val, 11, "a_val"); + ASSERT_EQ(skel->arena->b_val, 22, "b_val"); + +cleanup: + linked_arena__destroy(skel); +} + +static int link_objs(const char *out, const char *in1, const char *in2) +{ + struct bpf_linker *linker; + int err; + + linker = bpf_linker__new(out, NULL); + if (!ASSERT_OK_PTR(linker, "linker_new")) + return -EINVAL; + + err = bpf_linker__add_file(linker, in1, NULL); + if (!ASSERT_OK(err, in1)) + goto cleanup; + if (in2) { + err = bpf_linker__add_file(linker, in2, NULL); + if (!ASSERT_OK(err, in2)) + goto cleanup; + } + err = bpf_linker__finalize(linker); + ASSERT_OK(err, "finalize"); + +cleanup: + bpf_linker__free(linker); + return err; +} + +/* link in1 on its own first, then link the result with in2 */ +static void test_relink(const char *in1, const char *in2, const char *sec_name) +{ + char out1[] = "/tmp/linked_externs.XXXXXX", out2[] = "/tmp/linked_externs.XXXXXX"; + const struct btf_var_secinfo *vi; + struct bpf_object *obj = NULL; + const struct btf_type *t; + struct btf *btf; + int i, id, n; + + close(mkstemp(out1)); + close(mkstemp(out2)); + + if (!ASSERT_OK(link_objs(out1, in1, NULL), "link_stage1") || + !ASSERT_OK(link_objs(out2, out1, in2), "link_stage2")) + goto cleanup; + + obj = bpf_object__open_file(out2, NULL); + if (!ASSERT_OK_PTR(obj, "obj_open")) + goto cleanup; + + btf = bpf_object__btf(obj); + id = btf__find_by_name_kind(btf, sec_name, BTF_KIND_DATASEC); + if (!ASSERT_GT(id, 0, "find_datasec")) + goto cleanup; + + t = btf__type_by_id(btf, id); + vi = btf_var_secinfos(t); + n = btf_vlen(t); + for (i = 0; i < n; i++) { + t = btf__type_by_id(btf, vi[i].type); + ASSERT_NEQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_EXTERN, "var_resolved"); + } + + ASSERT_OK(bpf_object__load(obj), "obj_load"); + +cleanup: + bpf_object__close(obj); + unlink(out1); + unlink(out2); +} + +void test_linked_externs(void) +{ + if (test__start_subtest("skel_arena")) + test_skel(); + if (test__start_subtest("relink_arena")) + test_relink("linked_arena1.bpf.o", "linked_arena2.bpf.o", ".addr_space.1"); + if (test__start_subtest("relink_maps")) + test_relink("linked_maps1.bpf.o", "linked_maps2.bpf.o", ".maps"); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c new file mode 100644 index 0000000..7b7352d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c
@@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young <sean@mess.org> + +// A lirc chardev is a device representing a consumer IR (cir) device which +// can receive infrared signals from remote control and/or transmit IR. +// +// IR is sent as a series of pulses and space somewhat like morse code. The +// BPF program can decode this into scancodes so that rc-core can translate +// this into input key codes using the rc keymap. +// +// This test works by sending IR over rc-loopback, so the IR is processed by +// BPF and then decoded into scancodes. The lirc chardev must be the one +// associated with rc-loopback, see the output of ir-keytable(1). +// +// The following CONFIG options must be enabled for the test to succeed: +// CONFIG_RC_CORE=y +// CONFIG_BPF_LIRC_MODE2=y +// CONFIG_RC_LOOPBACK=y +// CONFIG_LIRC=y + +#include <linux/input.h> +#include <linux/lirc.h> +#include <glob.h> +#include <limits.h> +#include <poll.h> +#include <test_progs.h> +#include "lirc_mode2.skel.h" + +/* + * Read the DEVNAME= line out of the first uevent file that matches + * pattern, and turn it into a /dev/<name> path. + */ +static bool find_devname(const char *pattern, char *path, size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + FILE *f; + + if (glob(pattern, 0, NULL, &gl) || gl.gl_pathc == 0) + goto out; + + f = fopen(gl.gl_pathv[0], "r"); + if (!f) + goto out; + + char line[256]; + + while (fgets(line, sizeof(line), f)) { + char *val; + + if (strncmp(line, "DEVNAME=", 8)) + continue; + + val = line + 8; + val[strcspn(val, "\n")] = '\0'; + snprintf(path, path_sz, "/dev/%s", val); + found = true; + break; + } + + fclose(f); +out: + globfree(&gl); + return found; +} + +/* Load rc-loopback and find the lirc and input chardevs it created. */ +static bool find_loopback_devices(char *lirc_path, char *input_path, + size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + + /* Ignore failure, we check for the resulting devices below. */ + SYS_NOFAIL("modprobe rc-loopback > /dev/null 2>&1"); + + if (glob("/sys/class/rc/rc*", 0, NULL, &gl)) + goto out; + + for (size_t i = 0; i < gl.gl_pathc; i++) { + const char *rcdir = gl.gl_pathv[i]; + char uevent_path[PATH_MAX]; + char uevent[4096]; + char pattern[PATH_MAX]; + FILE *f; + size_t n; + + snprintf(uevent_path, sizeof(uevent_path), "%s/uevent", rcdir); + f = fopen(uevent_path, "r"); + if (!f) + continue; + n = fread(uevent, 1, sizeof(uevent) - 1, f); + fclose(f); + uevent[n] = '\0'; + + if (!strstr(uevent, "DRV_NAME=rc-loopback")) + continue; + + snprintf(pattern, sizeof(pattern), "%s/lirc*/uevent", rcdir); + if (!find_devname(pattern, lirc_path, path_sz)) + continue; + + snprintf(pattern, sizeof(pattern), "%s/input*/event*/uevent", rcdir); + if (!find_devname(pattern, input_path, path_sz)) + continue; + + found = true; + break; + } + +out: + if (!found) { + fprintf(stderr, "No rc devices found\n"); + fprintf(stderr, "Enable CONFIG_RC_LOOPBACK and CONFIG_BPF_LIRC_MODE2\n"); + } + + globfree(&gl); + return found; +} + +void test_lirc_mode2(void) +{ + char lirc_path[PATH_MAX], input_path[PATH_MAX]; + int lircfd = -1, inputfd = -1, progfd, progfd2 = -1; + struct lirc_mode2 *skel = NULL, *skel2 = NULL; + __u32 prog_ids[10], prog_flags[10], prog_cnt; + struct bpf_prog_info info; + __u32 info_len, prog_id, prog_id2; + int testir1 = 0x8ead; /* keydown flag (0x8000) | scancode 0xead */ + int testir2 = 0x4081; /* pointer_rel flag (0x4000) | rel_x=1 | rel_y=1 */ + struct input_event event; + struct pollfd pfd = {}; + int ret; + + if (getuid() != 0) { + test__skip(); + return; + } + + if (!find_loopback_devices(lirc_path, input_path, sizeof(lirc_path))) { + test__skip(); + return; + } + + skel = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel, "lirc_mode2__open_and_load")) + return; + + progfd = bpf_program__fd(skel->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd, &info, &info_len); + if (!ASSERT_OK(ret, "get first program's info")) + goto out; + prog_id = info.id; + + lircfd = open(lirc_path, O_RDWR | O_NONBLOCK); + if (!ASSERT_GE(lircfd, 0, "open lirc device")) + goto out; + + /* Try to detach it before it was ever attached, should fail. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_EQ(ret, -ENOENT, "detach unattached program")) + goto out; + + inputfd = open(input_path, O_RDONLY | O_NONBLOCK); + if (!ASSERT_GE(inputfd, 0, "open input device")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs before attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "no programs should be attached yet")) + goto out; + + /* Invalid attach flags must be rejected, and must not attach. */ + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 1); + if (!ASSERT_EQ(ret, -EINVAL, "attach with invalid flags")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after rejected attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "rejected attach should not attach")) + goto out; + + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach program to lirc device")) + goto out; + + /* Invalid query flags must be rejected too, without upsetting state. */ + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 1, prog_flags, prog_ids, + &prog_cnt); + ASSERT_EQ(ret, -EINVAL, "query with invalid flags"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after attach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should be attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "queried id should match attached program"); + + /* Write raw IR */ + ret = write(lircfd, &testir1, sizeof(testir1)); + if (!ASSERT_EQ(ret, sizeof(testir1), "send test IR message 1")) + goto out_detach; + + pfd.fd = inputfd; + pfd.events = POLLIN; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 1")) + goto out_detach; + + if (event.type == EV_MSC && event.code == MSC_SCAN && + event.value == 0xead) + break; + } + + /* Write raw IR */ + ret = write(lircfd, &testir2, sizeof(testir2)); + if (!ASSERT_EQ(ret, sizeof(testir2), "send test IR message 2")) + goto out_detach; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 2")) + goto out_detach; + + if (event.type == EV_REL && event.code == REL_Y && + event.value == 1) + break; + } + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after IR was decoded")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should still be attached")) + goto out_detach; + + /* + * The lirc chardev can hold more than one attached program at once. + * Load a second, independent instance and check it can be attached + * alongside the first, queried, and then detached on its own + * without disturbing the first program's attachment. + */ + skel2 = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel2, "lirc_mode2__open_and_load (2nd)")) + goto out_detach; + + progfd2 = bpf_program__fd(skel2->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd2, &info, &info_len); + if (!ASSERT_OK(ret, "get second program's info")) + goto out_detach; + prog_id2 = info.id; + + ret = bpf_prog_attach(progfd2, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach second program to lirc device")) + goto out_detach; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second attach")) + goto out_detach2; + if (!ASSERT_EQ(prog_cnt, 2, "two programs should be attached")) + goto out_detach2; + ASSERT_TRUE((prog_ids[0] == prog_id && prog_ids[1] == prog_id2) || + (prog_ids[0] == prog_id2 && prog_ids[1] == prog_id), + "queried ids should be the two attached programs"); + + /* Detach the second program; the first should remain attached. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_OK(ret, "detach second program")) + goto out_detach2; + + /* Detaching an already-detached program should now fail. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach second program again"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second detach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should remain attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "remaining program should be the first one"); + +out_detach: + /* Let's try detaching it now it is actually attached. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_OK(ret, "detach program from lirc device"); + + /* Detaching it again should now fail the same way. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach program from lirc device again"); + goto out; + +out_detach2: + /* Best-effort cleanup of the second program before bailing out. */ + bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + goto out_detach; + +out: + if (inputfd >= 0) + close(inputfd); + if (lircfd >= 0) + close(lircfd); + lirc_mode2__destroy(skel2); + lirc_mode2__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c b/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c new file mode 100644 index 0000000..b91c5c6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c
@@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <fcntl.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <sys/mount.h> +#include <sys/stat.h> +#include <sys/xattr.h> +#include <test_progs.h> + +#include "lsm_inode_init_xattr.skel.h" +#include "lsm_inode_init_xattr_budget.skel.h" +#include "lsm_inode_init_xattr_value.skel.h" + +#define INIT_XATTRS_MAX 2 +#define VALUE_SIZE_MAX (64 * 1024) +#define TESTDIR "/tmp/test_progs_lsm_init_xattr" +#define RAMFSDIR "/tmp/test_progs_lsm_init_xattr_ramfs" + +static bool testdir_mounted; +static bool ramfsdir_mounted; + +static int testdir_setup(void) +{ + if (mkdir(TESTDIR, 0755) && errno != EEXIST) + return -errno; + if (mount("tmpfs", TESTDIR, "tmpfs", 0, NULL)) + return -errno; + testdir_mounted = true; + if (mkdir(RAMFSDIR, 0755) && errno != EEXIST) + return -errno; + if (mount("ramfs", RAMFSDIR, "ramfs", 0, NULL)) + return -errno; + ramfsdir_mounted = true; + return 0; +} + +static void testdir_cleanup(void) +{ + if (ramfsdir_mounted) + umount(RAMFSDIR); + rmdir(RAMFSDIR); + if (testdir_mounted) + umount(TESTDIR); + rmdir(TESTDIR); +} + +static bool lsm_is_active(const char *name) +{ + char buf[512], *tok; + int fd, len; + + fd = open("/sys/kernel/security/lsm", O_RDONLY); + if (fd < 0) + return true; + len = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (len <= 0) + return true; + buf[len] = '\0'; + for (tok = strtok(buf, ",\n"); tok; tok = strtok(NULL, ",\n")) + if (!strcmp(tok, name)) + return true; + return false; +} + +static int read_label(const char *path, const char *name, char *buf, size_t sz) +{ + int ret = getxattr(path, name, buf, sz); + + return ret < 0 ? -errno : ret; +} + +static void assert_label(const char *path, const char *name, const char *want) +{ + char buf[64] = {}; + int ret; + + ret = read_label(path, name, buf, sizeof(buf)); + if (!ASSERT_EQ(ret, (int)strlen(want) + 1, name)) + return; + ASSERT_STREQ(buf, want, name); +} + +static struct lsm_inode_init_xattr *policy_attach(void) +{ + struct lsm_inode_init_xattr *skel; + + skel = lsm_inode_init_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return NULL; + + skel->bss->monitored_pid = getpid(); + if (!ASSERT_OK(lsm_inode_init_xattr__attach(skel), "skel_attach")) { + lsm_inode_init_xattr__destroy(skel); + return NULL; + } + return skel; +} + +static void test_init_labels(void) +{ + struct lsm_inode_init_xattr *skel; + const char *file = TESTDIR "/file"; + const char *subdir = TESTDIR "/subdir"; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_OK(skel->data->zone_err, "zone_err"); + ASSERT_OK(skel->data->origin_err, "origin_err"); + assert_label(file, "security.bpf.zone", "default"); + assert_label(file, "security.bpf.origin", "created"); + + if (!ASSERT_OK(mkdir(subdir, 0755), "mkdir")) + goto out; + assert_label(subdir, "security.bpf.zone", "default"); + assert_label(subdir, "security.bpf.origin", "created"); + rmdir(subdir); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_inherit_from_parent(void) +{ + const char *zonedir = TESTDIR "/zoned"; + const char *file = TESTDIR "/zoned/file"; + struct lsm_inode_init_xattr *skel; + int dirfd = -1, fd = -1, err; + struct { + char v[32]; + __u32 len; + } label = {}; + + if (!ASSERT_OK(mkdir(zonedir, 0755), "mkdir_zoned")) + return; + + skel = policy_attach(); + if (!skel) + goto out_dir; + + dirfd = open(zonedir, O_RDONLY | O_DIRECTORY); + if (!ASSERT_GE(dirfd, 0, "open_zoned")) + goto out; + + strncpy(label.v, "restricted", sizeof(label.v) - 1); + label.len = strlen("restricted") + 1; + + err = bpf_map_update_elem(bpf_map__fd(skel->maps.inode_zone), &dirfd, + &label, BPF_ANY); + if (!ASSERT_OK(err, "seed_parent_zone")) + goto out; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_EQ(skel->bss->inherited, 1, "inherited"); + ASSERT_OK(skel->data->zone_err, "zone_err"); + assert_label(file, "security.bpf.zone", "restricted"); + assert_label(file, "security.bpf.origin", "created"); +out: + if (fd >= 0) + close(fd); + if (dirfd >= 0) + close(dirfd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +out_dir: + rmdir(zonedir); +} + +static void test_refused_claims(void) +{ + const char *file = TESTDIR "/refused"; + struct lsm_inode_init_xattr *skel; + char buf[64]; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_EQ(skel->data->overflow_err, -ENOSPC, "overflow_err"); + ASSERT_EQ(read_label(file, "security.bpf.overflow", buf, sizeof(buf)), + -ENODATA, "overflow_absent"); + + ASSERT_EQ(skel->data->toolong_err, -EINVAL, "toolong_err"); + + ASSERT_EQ(skel->data->selinux_err, -EPERM, "selinux_err"); + ASSERT_EQ(skel->data->user_err, -EPERM, "user_err"); + + if (!lsm_is_active("selinux")) + ASSERT_EQ(read_label(file, "security.selinux", buf, sizeof(buf)), + -ENODATA, "selinux_absent"); + ASSERT_EQ(read_label(file, "user.zone", buf, sizeof(buf)), + -ENODATA, "user_absent"); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_null_xattrs(void) +{ + const char *file = RAMFSDIR "/file"; + struct lsm_inode_init_xattr *skel; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + ASSERT_EQ(skel->data->zone_err, -EOPNOTSUPP, "zone_err"); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_shared_budget(void) +{ + struct lsm_inode_init_xattr_budget *skel[INIT_XATTRS_MAX + 1] = {}; + struct bpf_link *link[INIT_XATTRS_MAX + 1] = {}; + const char *file = TESTDIR "/budget"; + int claimed = 0, refused = 0; + int i, fd = -1; + + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + skel[i] = lsm_inode_init_xattr_budget__open(); + if (!ASSERT_OK_PTR(skel[i], "skel_open")) + goto out; + + snprintf(skel[i]->rodata->xattr_name, + sizeof(skel[i]->rodata->xattr_name), + "security.bpf.slot%d", i); + + if (!ASSERT_OK(lsm_inode_init_xattr_budget__load(skel[i]), + "skel_load")) + goto out; + + skel[i]->bss->monitored_pid = getpid(); + + link[i] = bpf_program__attach_lsm(skel[i]->progs.claim_one); + if (!ASSERT_OK_PTR(link[i], "attach")) + goto out; + } + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + int err = skel[i]->data->claim_err; + + ASSERT_TRUE(skel[i]->bss->hook_ran, "hook_ran"); + if (!err) + claimed++; + else if (ASSERT_EQ(err, -ENOSPC, "claim_err")) + refused++; + } + + ASSERT_EQ(claimed, INIT_XATTRS_MAX, "claimed"); + ASSERT_EQ(refused, 1, "refused"); +out: + if (fd >= 0) + close(fd); + remove(file); + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + bpf_link__destroy(link[i]); + lsm_inode_init_xattr_budget__destroy(skel[i]); + } +} + +static int claim_value(const char *file, __u32 len, __u64 flags, + int hook_retval, int *open_errno) +{ + struct lsm_inode_init_xattr_value *skel; + struct bpf_link *link = NULL; + int err = 1, fd; + + skel = lsm_inode_init_xattr_value__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return 1; + + skel->rodata->value_len = len; + skel->rodata->dynptr_flags = flags; + skel->rodata->hook_retval = hook_retval; + + if (!ASSERT_OK(lsm_inode_init_xattr_value__load(skel), "skel_load")) + goto out; + + skel->bss->monitored_pid = getpid(); + + link = bpf_program__attach_lsm(skel->progs.claim_value); + if (!ASSERT_OK_PTR(link, "attach")) + goto out; + + fd = open(file, O_CREAT | O_RDWR | O_EXCL, 0644); + *open_errno = fd < 0 ? errno : 0; + if (fd >= 0) + close(fd); + + if (ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + err = skel->data->claim_err; +out: + bpf_link__destroy(link); + lsm_inode_init_xattr_value__destroy(skel); + return err; +} + +static void test_value_shapes(void) +{ + const char *file = TESTDIR "/value"; + char buf[64]; + int err, open_errno; + + remove(file); + + err = claim_value(file, 0, 0, 0, &open_errno); + ASSERT_OK(err, "empty_value_err"); + ASSERT_OK(open_errno, "empty_value_open"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + 0, "empty_value_len"); + remove(file); + + err = claim_value(file, sizeof(buf), 1, 0, &open_errno); + ASSERT_EQ(err, -EINVAL, "bad_dynptr_err"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENODATA, "bad_dynptr_absent"); + remove(file); + + err = claim_value(file, VALUE_SIZE_MAX + 1, 0, 0, &open_errno); + ASSERT_EQ(err, -E2BIG, "oversized_err"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENODATA, "oversized_absent"); + remove(file); + + err = claim_value(file, 8, 0, -EPERM, &open_errno); + ASSERT_OK(err, "denied_claim_err"); + ASSERT_EQ(open_errno, EPERM, "denied_open"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENOENT, "denied_absent"); + remove(file); +} + +void test_lsm_inode_init_xattr(void) +{ + if (!ASSERT_OK(testdir_setup(), "testdir_setup")) + goto out; + + if (test__start_subtest("init_labels")) + test_init_labels(); + if (test__start_subtest("inherit_from_parent")) + test_inherit_from_parent(); + if (test__start_subtest("refused_claims")) + test_refused_claims(); + if (test__start_subtest("null_xattrs")) + test_null_xattrs(); + if (test__start_subtest("shared_budget")) + test_shared_budget(); + if (test__start_subtest("value_shapes")) + test_value_shapes(); +out: + testdir_cleanup(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 39e8a3b..5c5560d 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c
@@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only +#include <net/if.h> #include <netinet/in.h> +#include <sys/ioctl.h> #include "network_helpers.h" #include "test_progs.h" @@ -141,6 +143,54 @@ static int set_bottom_addr(const char *ns1, const char *ns2, const char *ns3) return 1; } +/* + * A veth whose peer sits in another netns with the same ifindex gets its + * carrier-on handled as a non-urgent linkwatch event, i.e. up to 1s late. + * Until then IPv6 considers the link not ready (no ff00::/8 route, no + * link-local address) and silently drops neighbour solicitations, so wait + * for all links to be operationally up (IFF_RUNNING) before sending traffic. + */ +static int wait_for_oper_up(const char *ns, const char *dev) +{ + struct nstoken *nstoken; + struct ifreq ifr = {}; + int i, fd, ret = -1; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "open ns")) + return -1; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_OK_FD(fd, "socket")) + goto out; + + strncpy(ifr.ifr_name, dev, IFNAMSIZ - 1); + for (i = 0; i < 50; i++) { + if (!ASSERT_OK(ioctl(fd, SIOCGIFFLAGS, &ifr), "SIOCGIFFLAGS")) + break; + if (ifr.ifr_flags & IFF_RUNNING) { + ret = 0; + break; + } + usleep(100000); + } + close(fd); +out: + ASSERT_OK(ret, dev); + close_netns(nstoken); + return ret; +} + +static int wait_for_links(const char *ns1, const char *ns2, const char *ns3) +{ + if (wait_for_oper_up(ns1, "veth1") || wait_for_oper_up(ns1, "veth5") || + wait_for_oper_up(ns2, "veth2") || wait_for_oper_up(ns2, "veth3") || + wait_for_oper_up(ns2, "veth6") || wait_for_oper_up(ns2, "veth7") || + wait_for_oper_up(ns3, "veth4") || wait_for_oper_up(ns3, "veth8")) + return -1; + return 0; +} + static int configure_vrf(const char *ns1, const char *ns2) { if (!ns1 || !ns2) @@ -304,6 +354,9 @@ static int setup_network(char *ns1, char *ns2, char *ns3, const char *vrf) if (!ASSERT_OK(configure_ns3(ns3), "configure ns3 routes")) goto fail; + if (!ASSERT_OK(wait_for_links(ns1, ns2, ns3), "wait for links")) + goto fail; + /* Link bottom route to the GRE tunnels */ SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s %s", ns1, IP4_ADDR_GRE, IP4_ADDR_6, vrf);
diff --git a/tools/testing/selftests/bpf/prog_tests/map_excl.c b/tools/testing/selftests/bpf/prog_tests/map_excl.c index 3f4422b..9ff4cd3 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_excl.c +++ b/tools/testing/selftests/bpf/prog_tests/map_excl.c
@@ -161,12 +161,49 @@ static void test_map_excl_create_validation(void) ASSERT_EQ(fd, -EINVAL, "reject size with NULL hash pointer"); } +/* + * The exclusive map bound through a continuous fd_array rather than by an + * insn. The kernel resolves the array before it verifies the program, so it + * has to have the digest of the program at hand by then. + */ +static void test_map_excl_fd_array(void) +{ + struct map_excl *skel = map_excl__open(); + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int err, fd, map_fd; + + if (!ASSERT_OK_PTR(skel, "map_excl__open")) + return; + err = bpf_map__set_exclusive_program(skel->maps.excl_map, skel->progs.should_have_access); + if (!ASSERT_OK(err, "bpf_map__set_exclusive_program")) + goto out; + + bpf_program__set_autoload(skel->progs.should_have_access, true); + bpf_program__set_autoload(skel->progs.should_not_have_access, true); + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "map_excl prepare")) + goto out; + + map_fd = bpf_map__fd(skel->maps.excl_map); + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + fd = bpf_program__clone(skel->progs.should_have_access, &opts); + if (ASSERT_GE(fd, 0, "exclusive program with the map in fd_array")) + close(fd); + fd = bpf_program__clone(skel->progs.should_not_have_access, &opts); + ASSERT_EQ(fd, -EACCES, "other program with the map in fd_array"); +out: + map_excl__destroy(skel); +} + void test_map_excl(void) { if (test__start_subtest("map_excl_allowed")) test_map_excl_allowed(); if (test__start_subtest("map_excl_denied")) test_map_excl_denied(); + if (test__start_subtest("map_excl_fd_array")) + test_map_excl_fd_array(); if (test__start_subtest("map_excl_no_map_in_map")) test_map_excl_no_map_in_map(); if (test__start_subtest("map_excl_no_map_iter"))
diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c new file mode 100644 index 0000000..8d0ff30 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c
@@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <linux/filter.h> + +#define MAY_GOTO(off) BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, off, 0) +#define FILL_CNT 33000 + +static struct bpf_insn *fill(struct bpf_insn *insn, int cnt, int reg) +{ + while (cnt--) + *insn++ = BPF_MOV64_REG(BPF_REG_0, reg); + return insn; +} + +/* + * Syscall prog that loops until may_goto expires. Its target is 'off' insns + * away: + * + * r6 = r7 = 0, but not a constant for the verifier + * r8 = 2 + * r9 = 0 + * r0 = r8 x FILL_CNT or 1 + * 1: if r6 == r9 goto +1 + * exit + * r6 = r8 + * r0 = r9 x N when off is negative + * if r7 != r9 goto 3f + * 2: may_goto off (1b or 4f) + * goto 2b + * 3: r0 = r8 x N when off is positive + * 4: exit + * + * The prog returns 0 when may_goto jumps to its target and 2 when + * it lands in one of 'r0 = r8' areas. There are no constants between + * may_goto and its target to keep the distance when constants are blinded. + */ +static void test_far(int off) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, .prog_flags = BPF_F_SLEEPABLE); + LIBBPF_OPTS(bpf_test_run_opts, topts); + int front = off < 0 ? 1 : FILL_CNT; + int mid = off < 0 ? -off - 5 : 1; + int back = off < 0 ? FILL_CNT : off - 1; + struct bpf_insn *insns, *insn; + int fd, err; + + insns = calloc(front + mid + back + 16, sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "calloc")) + return; + + insn = insns; + *insn++ = BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns); + *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_0); + *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_6, 63); + *insn++ = BPF_MOV64_REG(BPF_REG_7, BPF_REG_0); + *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 63); + *insn++ = BPF_MOV64_IMM(BPF_REG_8, 2); + *insn++ = BPF_MOV64_IMM(BPF_REG_9, 0); + insn = fill(insn, front, BPF_REG_8); + *insn++ = BPF_JMP_REG(BPF_JEQ, BPF_REG_6, BPF_REG_9, 1); + *insn++ = BPF_EXIT_INSN(); + *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_8); + insn = fill(insn, mid, BPF_REG_9); + *insn++ = BPF_JMP_REG(BPF_JNE, BPF_REG_7, BPF_REG_9, 2); + *insn++ = MAY_GOTO(off); + *insn++ = BPF_JMP_A(-2); + insn = fill(insn, back, BPF_REG_8); + *insn++ = BPF_EXIT_INSN(); + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", insns, insn - insns, &opts); + free(insns); + if (!ASSERT_GE(fd, 0, "prog_load")) + return; + + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "retval"); + close(fd); +} + +void test_may_goto_far(void) +{ + if (test__start_subtest("32762")) + test_far(32762); + if (test__start_subtest("32767")) + test_far(32767); + if (test__start_subtest("-32767")) + test_far(-32767); + if (test__start_subtest("-32768")) + test_far(-32768); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c new file mode 100644 index 0000000..4f8476d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c
@@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "may_goto_priv_stack.skel.h" + +static void check_regs(__u64 *regs) +{ + int i; + + for (i = 0; i < 6; i++) + ASSERT_EQ(regs[i], 0, "reg"); + /* count is refreshed at least once when may_goto is timed */ + ASSERT_GT(regs[6], 0xffff, "iterations"); +} + +void test_may_goto_priv_stack(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct may_goto_priv_stack *skel; + int err; + + skel = may_goto_priv_stack__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + err = may_goto_priv_stack__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto out; + + memset(skel->bss->regs_64, 0xff, sizeof(skel->bss->regs_64)); + memset(skel->bss->regs_128, 0xff, sizeof(skel->bss->regs_128)); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.priv_stack_64), &topts); + if (!ASSERT_OK(err, "test_run")) + goto out; + + check_regs(skel->bss->regs_64); + check_regs(skel->bss->regs_128); +out: + may_goto_priv_stack__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c new file mode 100644 index 0000000..c22dcfa --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c
@@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <limits.h> +#include <linux/magic.h> +#include <sys/statfs.h> +#include <unistd.h> + +#include "cgroup_helpers.h" +#include "memcg_proactive_reclaim.skel.h" + +#define CG_PATH "/memcg_proactive_reclaim" + +/* + * Large enough that a single reclaim pass cannot come close to it, so that + * "reclaimed less than was asked for" is not page-size noise: one pass is + * capped at MEMCG_CHARGE_BATCH pages, which is 256 KiB on 4K pages but 4 MiB + * on 64K pages. + */ +#define FILE_SIZE (32 * 1024 * 1024UL) +#define BUF_SIZE (64 * 1024) + +struct reclaim_args { + __u64 cgroup_id; + __u64 size; +}; + +/* + * The data file has to sit on a writable regular filesystem: tmpfs pages are + * charged as shmem, so whether they can be reclaimed at all depends on swap + * being available, and a read-only mount cannot host the file at all (EROFS + * applies to root too). /tmp is tmpfs on many systems, and test_progs is + * routinely run from a tmpfs working directory, so both candidates need the + * check. + */ +static const char *workload_dir(void) +{ + static const char * const dirs[] = { "/tmp", "." }; + struct statfs st; + int i; + + for (i = 0; i < ARRAY_SIZE(dirs); i++) + if (!statfs(dirs[i], &st) && st.f_type != TMPFS_MAGIC && + st.f_type != RAMFS_MAGIC && !access(dirs[i], W_OK)) + return dirs[i]; + + return NULL; +} + +void test_memcg_proactive_reclaim(void) +{ + struct memcg_proactive_reclaim *skel = NULL; + struct reclaim_args args = {}; + + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &args, + .ctx_size_in = sizeof(args)); + + char data_file[PATH_MAX]; + static char buf[BUF_SIZE]; + __u64 cgroup_id; + const char *dir; + off_t off; + int cg_fd = -1, data_fd = -1, err; + + /* + * Both candidate directories can be unusable: tmpfs (initramfs-based + * CI, for instance) or a read-only mount. That is an environment + * limitation, not a test failure. + */ + dir = workload_dir(); + if (!dir) { + test__skip(); + return; + } + + snprintf(data_file, sizeof(data_file), + "%s/memcg_proactive_reclaim_XXXXXX", dir); + data_fd = mkstemp(data_file); + if (!ASSERT_GE(data_fd, 0, "mkstemp")) + return; + + cg_fd = cgroup_setup_and_join(CG_PATH); + if (!ASSERT_OK_FD(cg_fd, "cgroup_setup_and_join")) + goto out; + + cgroup_id = get_cgroup_id(CG_PATH); + if (!ASSERT_GT(cgroup_id, 0, "get_cgroup_id")) + goto out; + + skel = memcg_proactive_reclaim__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto out; + + /* Charge FILE_SIZE of clean page cache to the cgroup. */ + for (off = 0; off < (off_t)FILE_SIZE; off += sizeof(buf)) + if (!ASSERT_EQ(write(data_fd, buf, sizeof(buf)), + (ssize_t)sizeof(buf), "write")) + goto out; + if (!ASSERT_OK(fsync(data_fd), "fsync")) + goto out; + + args.cgroup_id = cgroup_id; + args.size = FILE_SIZE; + skel->bss->reclaimed = 0; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.memcg_proactive_reclaim), + &opts); + if (!ASSERT_OK(err, "test_run")) + goto out; + if (!ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + + /* + * A single call is a single bounded pass: it reclaims something, but + * stops well short of the requested size instead of retrying until the + * goal is reached the way a write to memory.reclaim does. + */ + ASSERT_GT(skel->bss->reclaimed, 0, "reclaimed"); + ASSERT_LT(skel->bss->reclaimed, (__s64)FILE_SIZE, "single pass"); + +out: + if (skel) + memcg_proactive_reclaim__destroy(skel); + if (cg_fd >= 0) + close(cg_fd); + if (data_fd >= 0) { + close(data_fd); + unlink(data_file); + } + cleanup_cgroup_environment(); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/missed.c b/tools/testing/selftests/bpf/prog_tests/missed.c index ed8857a..b5fb0267 100644 --- a/tools/testing/selftests/bpf/prog_tests/missed.c +++ b/tools/testing/selftests/bpf/prog_tests/missed.c
@@ -36,7 +36,7 @@ static void test_missed_perf_kprobe(void) ASSERT_EQ(info.type, BPF_LINK_TYPE_PERF_EVENT, "info.type"); ASSERT_EQ(info.perf_event.type, BPF_PERF_EVENT_KPROBE, "info.perf_event.type"); - ASSERT_EQ(info.perf_event.kprobe.missed, 1, "info.perf_event.kprobe.missed"); + ASSERT_GE(info.perf_event.kprobe.missed, 1, "info.perf_event.kprobe.missed"); cleanup: missed_kprobe__destroy(skel); @@ -120,9 +120,9 @@ static void test_missed_tp_recursion(void) ASSERT_EQ(topts.retval, 0, "test_run"); ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test1)), 0, "test1_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test2)), 1, "test2_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test2)), 1, "test2_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); cleanup: missed_tp_recursion__destroy(skel);
diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index 7b4a1e2..67124dd 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c
@@ -449,6 +449,95 @@ static void test_lru_percpu_hash_cpu_flag_create(void) test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); } +static bool verify_map_value_all_cpus(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem all_cpus")) + return false; + + for (i = 0 ; i < nr_cpus ; i++) { + if (!ASSERT_EQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value all_cpus")) + return false; + } + + return true; +} + +static bool verify_map_value_first_cpu(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem cpu")) + return false; + + if (!ASSERT_EQ(*(u32 *)values, expected_value, "bpf_map_lookup_elem value cpu")) + return false; + + for (i = 1 ; i < nr_cpus ; i++) { + if (!ASSERT_NEQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value cpu")) + return false; + } + + return true; +} + +static void test_percpu_cgroup_storage_flags_combination(struct bpf_map *map, int nr_cpus, + struct bpf_cgroup_storage_key *key) +{ + size_t value_sz = sizeof(u32), key_sz, elem_sz; + u32 *values, test_value = 0xAABBCCDD; + int err, map_fd; + u64 flags; + + key_sz = sizeof(*key); + elem_sz = roundup(value_sz, 8); + map_fd = bpf_map__fd(map); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map__update_elem(map, key, key_sz, values, elem_sz * nr_cpus, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, key, values, elem_sz, test_value, nr_cpus)) + goto out; + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem cpu|exist")) + goto out; + + verify_map_value_first_cpu(map_fd, key, values, elem_sz, test_value, nr_cpus); +out: + free(values); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -489,6 +578,7 @@ static void test_percpu_cgroup_storage_cpu_flag(void) goto out; test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false); + test_percpu_cgroup_storage_flags_combination(map, nr_cpus, &key); out: bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS); close(cgroup); @@ -537,6 +627,164 @@ static void test_hash_cpu_flag(void) test_map_op_cpu_flag(BPF_MAP_TYPE_HASH); } +static void test_percpu_map_flags_combination(enum bpf_map_type map_type) +{ + u32 max_entries = 3, key = 0, first_value = 0xAABBCCDD, test_value = first_value; + size_t value_sz = sizeof(u32), elem_sz; + int err, map_fd, nr_cpus; + bool is_hash_map; + u32 *values; + u64 flags; + + is_hash_map = (map_type == BPF_MAP_TYPE_PERCPU_HASH || + map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + elem_sz = roundup(value_sz, 8); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + map_fd = bpf_map_create(map_type, "test_flags_combination_map", + sizeof(u32), value_sz, max_entries, NULL); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) { + free(values); + return; + } + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map_update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + values[0] = test_value; + key = 0; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + values[0] = --test_value; + key = 1; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|noexist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = --test_value; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem all_cpus|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem cpu|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|exist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + if (map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) + goto out; + + /* Percpu LRU hash map should not delete old entries unnecessarily */ + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|exist")) + goto out; + + key = 0; + verify_map_value_all_cpus(map_fd, &key, values, elem_sz, first_value, nr_cpus); + +out: + close(map_fd); + free(values); +} + +static bool pin_current_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int err, cpu; + + err = sched_getaffinity(0, sizeof(*old_mask), old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + return false; + + cpu = sched_getcpu(); + if (!ASSERT_GE(cpu, 0, "sched_getcpu")) + return false; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + return ASSERT_OK(err, "sched_setaffinity"); +} + +static void test_percpu_hash_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_HASH); +} + +static void test_lru_percpu_hash_flags_combination(void) +{ + cpu_set_t old_mask; + + if (!pin_current_cpu(&old_mask)) + return; + + test_percpu_map_flags_combination(BPF_MAP_TYPE_LRU_PERCPU_HASH); + + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} + +static void test_percpu_array_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_ARRAY); +} + void test_percpu_alloc(void) { if (test__start_subtest("array")) @@ -565,4 +813,10 @@ void test_percpu_alloc(void) test_array_cpu_flag(); if (test__start_subtest("cpu_flag_hash")) test_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_combination_percpu_hash")) + test_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_lru_percpu_hash")) + test_lru_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_percpu_array")) + test_percpu_array_flags_combination(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index 7607cfc..ddb6cdc 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
@@ -13,6 +13,7 @@ static void clear_test_state(struct test_state *state) void test_prog_tests_framework(void) { struct test_state *state = env.test_state; + int subtest_cnt; /* in all the ASSERT calls below we need to return on the first * error due to the fact that we are cleaning the test state after @@ -30,25 +31,30 @@ void test_prog_tests_framework(void) return; clear_test_state(state); + subtest_cnt = 0; if (test__start_subtest("test_skip_subtest")) { test__skip(); test__end_subtest(); + subtest_cnt++; } if (test__start_subtest("test_skip_subtest")) { test__skip(); test__end_subtest(); + subtest_cnt++; } - if (!ASSERT_EQ(state->skip_cnt, 2, "skip_cnt_check")) + if (!ASSERT_EQ(state->skip_cnt, subtest_cnt, "skip_cnt_check")) return; if (!ASSERT_EQ(state->subtest_num, 3, "subtest_num_check")) return; clear_test_state(state); + subtest_cnt = 0; if (test__start_subtest("test_fail_subtest")) { test__fail(); test__end_subtest(); + subtest_cnt++; } - if (!ASSERT_EQ(state->error_cnt, 1, "error_cnt_check")) + if (!ASSERT_EQ(state->error_cnt, subtest_cnt, "error_cnt_check")) return; if (!ASSERT_EQ(state->subtest_num, 4, "subtest_num_check")) return; @@ -163,6 +169,12 @@ void test_prog_tests_framework_expected_msgs(void) for (i = 0; i < ARRAY_SIZE(cases); i++) { if (test__start_subtest(cases[i].name)) { + if (!env.subtest_state->stdout_saved) { + test__skip(); + test__end_subtest(); + continue; + } + error_cnt = env.subtest_state->error_cnt; msgs.patterns = cases[i].pats; msgs.cnt = 0; @@ -179,3 +191,30 @@ void test_prog_tests_framework_expected_msgs(void) } } } + +void test_prog_tests_framework_compare_text(void) +{ + int err; + + if (test__start_subtest("compare_text_match")) { + err = compare_text_to_expected("same\n", "same\n"); + ASSERT_EQ(err, 0, "match_rc"); + test__end_subtest(); + } + + if (test__start_subtest("compare_text_mismatch")) { + if (!env.subtest_state->stdout_saved) { + test__skip(); + } else { + err = compare_text_to_expected("line two\n", "line one\n"); + fflush(stdout); + + ASSERT_EQ(err, -1, "mismatch_rc"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one", + "diff_has_expected"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two", + "diff_has_actual"); + } + test__end_subtest(); + } +}
diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c index 4144132..a01a0bc 100644 --- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c +++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c
@@ -2,6 +2,8 @@ #include <test_progs.h> #include <network_helpers.h> +#define U32_MAX ((u32)UINT_MAX) + enum { QUEUE, STACK, @@ -101,8 +103,35 @@ static void test_queue_stack_map_by_type(int type) bpf_object__close(obj); } +static void test_queue_stack_map_alloc_check(void) +{ + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_oversize"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1, U32_MAX, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_u32max"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_STACK, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "stack_oversize"); + if (fd >= 0) + close(fd); + + /* A normal-sized map must still be created successfully. */ + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 64, 100, NULL); + ASSERT_GE(fd, 0, "queue_normal"); + if (fd >= 0) + close(fd); +} + void test_queue_stack_map(void) { test_queue_stack_map_by_type(QUEUE); test_queue_stack_map_by_type(STACK); + test_queue_stack_map_alloc_check(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c index 3dbcc09..3b4b233 100644 --- a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c
@@ -6,6 +6,7 @@ #include <stdbool.h> #include <string.h> #include <errno.h> +#include <error.h> #include <assert.h> #include <fcntl.h> #include <linux/bpf.h> @@ -21,6 +22,7 @@ #include "bpf_util.h" #include "test_progs.h" +#include "socket_helpers.h" #include "test_select_reuseport_common.h" #define MAX_TEST_NAME 80 @@ -740,18 +742,6 @@ static const char *maptype_str(enum bpf_map_type type) } } -static const char *family_str(sa_family_t family) -{ - switch (family) { - case AF_INET: - return "IPv4"; - case AF_INET6: - return "IPv6"; - default: - return "unknown"; - } -} - static const char *sotype_str(int sotype) { switch (sotype) { @@ -794,7 +784,7 @@ static void test_config(int sotype, sa_family_t family, bool inany) snprintf(s, sizeof(s), "%s %s/%s %s %s", maptype_str(inner_map_type), - family_str(family), sotype_str(sotype), + inet_family_str(family), sotype_str(sotype), inany ? "INANY" : "LOOPBACK", t->name); if (!test__start_subtest(s))
diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 77381d3..e43dba8f 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c
@@ -32,11 +32,22 @@ enum { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; -static int load_loader(const void *insns, __u32 insns_sz, int map_fd, - const void *sig, __u32 sig_sz, __s32 keyring_id, - __u32 fd_array_cnt) +#ifndef KEY_SPEC_BPF_KEYRING +#define KEY_SPEC_BPF_KEYRING -9 +#endif + +/* verify_sig_setup.sh exits with this when openssl cannot do ML-DSA. */ +#define SETUP_SKIP (-77) + +/* FIPS-204 ML-DSA-87 signature size, see include/crypto/mldsa.h. */ +#define MLDSA87_SIGNATURE_SIZE 4627 + +static int load_loader_log(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt, char *log_buf, __u32 log_sz) { union bpf_attr attr; int fd; @@ -48,18 +59,31 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd, attr.license = ptr_to_u64("Dual BSD/GPL"); attr.prog_flags = BPF_F_SLEEPABLE; attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = fd_array_cnt; if (sig) { attr.signature = ptr_to_u64(sig); attr.signature_size = sig_sz; attr.keyring_id = keyring_id; } - attr.fd_array_cnt = fd_array_cnt; + if (log_buf) { + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_sz; + } memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); return fd < 0 ? -errno : fd; } +static int load_loader(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) +{ + return load_loader_log(insns, insns_sz, map_fd, sig, sig_sz, keyring_id, + fd_array_cnt, NULL, 0); +} + static int run_gen_loader(const void *insns, __u32 insns_sz, const void *data, __u32 data_sz, const void *excl, __u32 excl_sz, @@ -156,12 +180,30 @@ static int run_setup(const char *cmd, const char *dir) } if (waitpid(pid, &status, 0) < 0) return -errno; - return (WIFEXITED(status) && - WEXITSTATUS(status) == 0) ? 0 : -EINVAL; + if (!WIFEXITED(status)) + return -EINVAL; + return -WEXITSTATUS(status); } -static int sign_buf(const char *dir, const void *buf, __u32 len, - void *sig, __u32 *sig_sz) +static void genkey_dir_fini(const char *dir) +{ + static const char * const files[] = { + "signing_key.der", "signing_key.pem", "x509.genkey", + }; + char path[PATH_MAX]; + size_t i; + + if (!dir) + return; + for (i = 0; i < ARRAY_SIZE(files); i++) { + snprintf(path, sizeof(path), "%s/%s", dir, files[i]); + unlink(path); + } + rmdir(dir); +} + +static int sign_buf_digest(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz, const char *digest) { char data_tmpl[PATH_MAX], key[PATH_MAX]; char sigpath[PATH_MAX + sizeof(".p7s")]; @@ -176,6 +218,7 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, fd = mkstemp(data_tmpl); if (fd < 0) return -errno; + snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (write(fd, buf, len) != (ssize_t)len) { close(fd); ret = -EIO; @@ -190,7 +233,7 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, } if (pid == 0) { snprintf(key, sizeof(key), "%s/signing_key.pem", dir); - execlp("./sign-file", "./sign-file", "-d", "sha256", + execlp("./sign-file", "./sign-file", "-d", digest, key, key, data_tmpl, NULL); exit(1); } @@ -200,34 +243,38 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, goto out; } - snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (stat(sigpath, &st) < 0) { ret = -errno; goto out; } if (st.st_size > (off_t)*sig_sz) { ret = -E2BIG; - goto out_sig; + goto out; } fd = open(sigpath, O_RDONLY); if (fd < 0) { ret = -errno; - goto out_sig; + goto out; } if (read(fd, sig, st.st_size) != st.st_size) { close(fd); ret = -EIO; - goto out_sig; + goto out; } close(fd); *sig_sz = st.st_size; -out_sig: - unlink(sigpath); out: + unlink(sigpath); unlink(data_tmpl); return ret; } +static int sign_buf(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz) +{ + return sign_buf_digest(dir, buf, len, sig, sig_sz, "sha256"); +} + struct gen_loader_fixture { struct test_signed_loader *skel; struct gen_loader_opts gopts; @@ -332,6 +379,51 @@ static void signature_enforced(void) gen_loader_fixture_fini(&f); } +/* + * CO-RE relocations would have the kernel rewrite the insns after the + * signature was checked over them, with values the loader picks through the + * BTF it supplies. A signed load that carries any is rejected before the + * signature is examined, so junk serves as one. + */ +static void signed_core_relo_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_core_relo relo = { + .insn_off = 0, + .type_id = 1, + .kind = BPF_CORE_TYPE_ID_LOCAL, + }; + char log_buf[1024] = {}; + union bpf_attr attr; + int fd; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.core_relos = ptr_to_u64(&relo); + attr.core_relo_cnt = 1; + attr.core_relo_rec_size = sizeof(relo); + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); + fd = fd < 0 ? -errno : fd; + ASSERT_EQ(fd, -EACCES, "signed load with CO-RE relocations rejected"); + ASSERT_HAS_SUBSTR(log_buf, "signed program cannot carry CO-RE relocations", "log"); + if (fd >= 0) + close(fd); +} + static void signed_nonexcl_fd_array_rejected(void) { static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; @@ -457,7 +549,7 @@ static void signed_btf_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -529,7 +621,6 @@ static void signature_failure_logs(void) static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; char log_buf[1024] = {}; struct gen_loader_fixture f; - union bpf_attr attr; int fd; if (gen_loader_fixture_init(&f) == 0) { @@ -538,22 +629,9 @@ static void signature_failure_logs(void) * failure is reported through the verifier log. A present-but- * invalid signature is rejected and the log says why. */ - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SYSCALL; - attr.insns = ptr_to_u64(f.gopts.insns); - attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); - attr.license = ptr_to_u64("Dual BSD/GPL"); - attr.prog_flags = BPF_F_SLEEPABLE; - attr.signature = ptr_to_u64(junk); - attr.signature_size = sizeof(junk); - attr.keyring_id = KEY_SPEC_SESSION_KEYRING; - attr.log_level = 1; - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = sizeof(log_buf); - memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); - - fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, - offsetofend(union bpf_attr, keyring_id)); + fd = load_loader_log(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0, + log_buf, sizeof(log_buf)); ASSERT_LT(fd, 0, "invalid signature rejected at load"); if (fd >= 0) close(fd); @@ -571,8 +649,9 @@ static void signature_too_large(void) if (gen_loader_fixture_init(&f) == 0) { /* - * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) - * is rejected before the buffer is read. + * signature_size beyond the kernel's bound + * (BPF_PROG_MAX_SIGNATURE_SIZE) is rejected before the buffer + * is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); @@ -626,6 +705,280 @@ static void signature_bad_keyring(void) gen_loader_fixture_fini(&f); } +static bool keyring_unsealed_boot(void) +{ + char val = 0; + int fd; + + fd = open("/sys/module/bpf/parameters/keyring_unsealed", O_RDONLY); + if (fd < 0) + return false; + if (read(fd, &val, 1) != 1) + val = 0; + close(fd); + return val == 'Y' || val == '1'; +} + +static int bpf_keyring_lookup(int *nr_keys) +{ + char line[512], type[32], desc[64]; + int serial = -ENOENT; + FILE *f; + + f = fopen("/proc/keys", "r"); + if (!f) + return -errno; + + while (fgets(line, sizeof(line), f)) { + unsigned int hex; + char *sum; + + if (sscanf(line, "%x %*s %*s %*s %*s %*s %*s %31s %63s", + &hex, type, desc) != 3) + continue; + if (strcmp(type, "keyring") || strcmp(desc, ".bpf:")) + continue; + + serial = (int)hex; + if (nr_keys) { + sum = strstr(line, ".bpf: "); + *nr_keys = !sum || !strncmp(sum + 6, "empty", 5) ? + 0 : atoi(sum + 6); + } + break; + } + fclose(f); + return serial; +} + +static long keyctl_ret(int cmd, unsigned long arg2, unsigned long arg3) +{ + long ret = syscall(__NR_keyctl, cmd, arg2, arg3); + + return ret < 0 ? -errno : ret; +} + +/* + * What the bpf keyring still needs once it got provisioned: KEY_POS_SEARCH + * for the in-kernel search during verification, and the user view/read bits + * so it stays visible in /proc/keys, rest is dropped so the enrolled is + * therefore final. + */ +#define BPF_KEYRING_PERM_LOCKED 0x08030000 +/* What bpf_keyring_init() grants at boot. */ +#define BPF_KEYRING_PERM_INITIAL 0x082f0000 + +static void bpf_keyring_sealed(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int serial, key, fd; + + if (keyring_unsealed_boot()) { + printf("%s:SKIP:the bpf keyring was unsealed at boot\n", __func__); + test__skip(); + return; + } + serial = bpf_keyring_lookup(NULL); + if (serial >= 0) { + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, + KEY_SPEC_BPF_KEYRING, 0), serial, + "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); + key = syscall(__NR_add_key, "user", "sealprobe", "x", 1, + KEY_SPEC_BPF_KEYRING); + if (key >= 0) + syscall(__NR_keyctl, KEYCTL_UNLINK, key, + KEY_SPEC_BPF_KEYRING); + ASSERT_EQ(key < 0 ? -errno : 0, -EPERM, + "nothing links into a sealed keyring"); + } + if (gen_loader_fixture_init(&f) == 0) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_BPF_KEYRING, 0); + ASSERT_EQ(fd, -ENOKEY, "sealed bpf keyring rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static int try_load(const struct gen_loader_fixture *f, const void *sig, + __u32 sig_sz, __s32 keyring_id, char *log_buf, __u32 log_sz) +{ + int map_fd, prog_fd; + + map_fd = setup_meta_map(f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + return map_fd; + prog_fd = load_loader_log(f->gopts.insns, f->gopts.insns_sz, map_fd, + sig, sig_sz, keyring_id, 1, log_buf, log_sz); + close(map_fd); + if (prog_fd >= 0) + close(prog_fd); + return prog_fd; +} + +/* + * This needs bpf.keyring_unsealed=1 on the guest kernel command line, which + * vmtest.sh can pass via KERNEL_CMDLINE_EXTRA. There is no way to unseal the + * keyring from here, so without it the test skips. It also only works once + * per boot, as restricting a keyring cannot be undone. + */ +static void bpf_keyring_provisioned(void) +{ + char dir_tmpl[] = "/tmp/bpfkeyringXXXXXX"; + char bad_tmpl[] = "/tmp/bpfkeyringbadXXXXXX"; + __u8 *sig = NULL, *bad = NULL, *buf = NULL; + int serial, err; + int nr_keys = 0, der_fd = -1; + struct gen_loader_fixture f; + __u32 sig_sz = 8192, bad_sz; + bool have_fixture = false; + char *dir, *bad_dir = NULL; + char log_buf[1024] = {}; + char path[PATH_MAX]; + __u8 der[4096]; + ssize_t der_sz; + + serial = bpf_keyring_lookup(&nr_keys); + if (serial < 0) { + printf("%s:SKIP:no bpf keyring (needs CONFIG_KEYS)\n", __func__); + test__skip(); + return; + } + if (nr_keys != 0) { + printf("%s:SKIP:the bpf keyring has already been provisioned\n", + __func__); + test__skip(); + return; + } + + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("genkey", dir), "verify_sig_setup genkey")) + goto rmdir; + + snprintf(path, sizeof(path), "%s/signing_key.der", dir); + der_fd = open(path, O_RDONLY); + if (!ASSERT_OK_FD(der_fd, "open signing_key.der")) + goto rmdir; + der_sz = read(der_fd, der, sizeof(der)); + close(der_fd); + if (!ASSERT_GT(der_sz, 0, "read signing_key.der")) + goto rmdir; + + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + serial, "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); + + err = syscall(__NR_add_key, "asymmetric", "", der, (size_t)der_sz, + KEY_SPEC_BPF_KEYRING); + if (err < 0 && errno == EPERM) { + printf("%s:SKIP:the bpf keyring is sealed, need bpf.keyring_unsealed=1\n", + __func__); + test__skip(); + goto rmdir; + } + if (!ASSERT_GE(err, 0, "add the signing key to the bpf keyring")) + goto rmdir; + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + -ENOKEY, "unrestricted keyring still not consulted"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused before provisioning"); + + if (!ASSERT_OK(syscall(__NR_keyctl, KEYCTL_RESTRICT_KEYRING, + KEY_SPEC_BPF_KEYRING, NULL, NULL), + "restrict bpf keyring")) + goto out; + + if (!ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, + NULL, 0), + "load signed by a key in the .bpf keyring")) + goto out; + + bad_dir = mkdtemp(bad_tmpl); + if (!ASSERT_OK_PTR(bad_dir, "mkdtemp unenrolled")) + goto out; + if (!ASSERT_OK(run_setup("genkey", bad_dir), "verify_sig_setup genkey unenrolled")) + goto out; + bad_sz = 8192; + bad = malloc(bad_sz); + if (!ASSERT_OK_PTR(bad, "bad sig buf")) + goto out; + if (!ASSERT_OK(sign_buf(bad_dir, buf, f.gopts.insns_sz + f.data_sz, bad, + &bad_sz), "sign with an unenrolled key")) + goto out; + + ASSERT_EQ(try_load(&f, bad, bad_sz, KEY_SPEC_BPF_KEYRING, log_buf, + sizeof(log_buf)), -ENOKEY, + "key outside the bpf keyring refused"); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "the bpf keyring was consulted"); + + f.blob[0] ^= 0xff; + err = try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0); + f.blob[0] ^= 0xff; + ASSERT_EQ(err, -EKEYREJECTED, "tampered metadata refused"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused once .bpf is in use"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -ENOENT, "keyring writable while the user bits are there"); + + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_LOCKED); + if (!ASSERT_OK(err, "drop the user bits on the bpf keyring")) + goto out; + + ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + "load still verified against the locked keyring"); + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + -EACCES, "the special id grants no rights of its own"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -EACCES, "unlink refused"); + err = keyctl_ret(KEYCTL_CLEAR, serial, 0); + ASSERT_EQ(err, -EACCES, "clear refused"); + err = keyctl_ret(KEYCTL_REVOKE, serial, 0); + ASSERT_EQ(err, -EACCES, "revoke refused"); + err = keyctl_ret(KEYCTL_INVALIDATE, serial, 0); + ASSERT_EQ(err, -EACCES, "invalidate refused"); + err = keyctl_ret(KEYCTL_SET_TIMEOUT, serial, 1); + ASSERT_EQ(err, -EACCES, "timeout refused"); + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_INITIAL); + ASSERT_EQ(err, -EACCES, "the bits cannot be granted back"); + + ASSERT_EQ(bpf_keyring_lookup(&nr_keys), serial, "keyring still there"); + ASSERT_EQ(nr_keys, 1, "the enrolled key survived"); +out: + if (have_fixture) + gen_loader_fixture_fini(&f); + genkey_dir_fini(bad_dir); + free(buf); + free(bad); + free(sig); +rmdir: + genkey_dir_fini(dir); +} + /* * A signed loader must ignore ctx-supplied map dimensions: the host cannot * resize a signed program's maps via the loader ctx. Drive a one-map program @@ -831,7 +1184,7 @@ static void signature_authenticates_insns(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -931,7 +1284,7 @@ static void signature_authenticates_metadata(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1267,7 +1620,7 @@ static void lsm_signature_verdict(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); dir = NULL; goto out; @@ -1450,7 +1803,7 @@ static void loadtime_verify(struct bpf_object *obj, int expect_maps) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1525,6 +1878,89 @@ static void loadtime_with_map(void) } /* + * End-to-end signed load with a post-quantum key. ML-DSA (FIPS-204) is wired + * through the X.509 and PKCS#7 parsers, and BPF reaches them via + * verify_pkcs7_signature() without knowing the algorithm, so an ML-DSA key in + * the keyring should verify an ML-DSA signed program with no BPF-side work. + */ +static void mldsa_signed_load(void) +{ + char dir_tmpl[] = "/tmp/bpfmldsaXXXXXX"; + int map_fd = -1, prog_fd = -1, err; + __u8 *sig = NULL, *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + __u32 sig_sz = 16384; + char *dir; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + + err = run_setup("setup-mldsa", dir); + if (err == SETUP_SKIP) { + printf("%s:SKIP:no working ML-DSA signing, set SELFTESTS_VERBOSE=1\n", + __func__); + test__skip(); + genkey_dir_fini(dir); + return; + } + if (!ASSERT_OK(err, "verify_sig_setup setup-mldsa")) { + genkey_dir_fini(dir); + return; + } + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + + /* + * ML-DSA hashes the message itself, but openssl before 4.0 cannot + * produce a CMS message without signedAttrs for it, and with those in + * play only SHA-512 is permitted for the messageDigest attribute. + */ + if (!ASSERT_OK(sign_buf_digest(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &sig_sz, "sha512"), + "sign insns||metadata with ML-DSA")) + goto out; + + /* + * Guard against the setup silently handing back some other key type: + * an RSA or ECDSA signature is a few hundred bytes, where an ML-DSA-87 + * one cannot be smaller than the raw signature it carries. + */ + ASSERT_GT(sig_sz, MLDSA87_SIGNATURE_SIZE, "ML-DSA-87 signature size"); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + sig_sz, KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_OK_FD(prog_fd, "ML-DSA signed loader load"); +out: + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + free(buf); + free(sig); + run_setup("cleanup", dir); +} + +/* * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL * program with no fd_array is signed over its instructions alone: the kernel * verifies the signature, folds no metadata, and the program loads. Exercise @@ -1548,7 +1984,7 @@ static void signed_no_fd_array(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1619,7 +2055,7 @@ static void signed_map_by_fd_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1681,7 +2117,7 @@ static void signed_sparse_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1735,7 +2171,7 @@ static void signed_module_kfunc_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1777,64 +2213,72 @@ static void signed_module_kfunc_rejected(void) run_setup("cleanup", dir); } +enum subtest_boot { + BOOT_ANY, + BOOT_SEALED, + BOOT_UNSEALED, +}; + +static const struct { + const char *name; + void (*fn)(void); + enum subtest_boot boot; +} subtests[] = { + { "loadtime_no_map", loadtime_no_map, BOOT_SEALED }, + { "loadtime_with_map", loadtime_with_map, BOOT_SEALED }, + { "metadata_match", metadata_match, BOOT_ANY }, + { "signature_enforced", signature_enforced, BOOT_SEALED }, + { "signed_nonexcl_fd_array_rejected", signed_nonexcl_fd_array_rejected, BOOT_SEALED }, + { "signed_unfrozen_fd_array_rejected", signed_unfrozen_fd_array_rejected, BOOT_SEALED }, + { "signed_nonarray_fd_array_rejected", signed_nonarray_fd_array_rejected, BOOT_SEALED }, + { "signed_btf_fd_array_rejected", signed_btf_fd_array_rejected, BOOT_ANY }, + { "signed_core_relo_rejected", signed_core_relo_rejected, BOOT_ANY }, + { "signed_module_kfunc_rejected", signed_module_kfunc_rejected, BOOT_SEALED }, + { "signature_failure_logs", signature_failure_logs, BOOT_SEALED }, + { "signature_too_large", signature_too_large, BOOT_ANY }, + { "signature_zero_size", signature_zero_size, BOOT_ANY }, + { "signature_bad_keyring", signature_bad_keyring, BOOT_SEALED }, + { "bpf_keyring_sealed", bpf_keyring_sealed, BOOT_ANY }, + { "mldsa_signed_load", mldsa_signed_load, BOOT_SEALED }, + { "metadata_ctx_max_entries_ignored", metadata_ctx_max_entries_ignored, BOOT_ANY }, + { "metadata_ctx_initial_value_ignored", metadata_ctx_initial_value_ignored, BOOT_ANY }, + { "signature_authenticates_insns", signature_authenticates_insns, BOOT_SEALED }, + { "signature_authenticates_metadata", signature_authenticates_metadata, BOOT_SEALED }, + { "hash_requires_frozen", hash_requires_frozen, BOOT_ANY }, + { "no_update_after_freeze", no_update_after_freeze, BOOT_ANY }, + { "freeze_writable_mmap", freeze_writable_mmap, BOOT_ANY }, + { "no_writable_mmap_frozen", no_writable_mmap_frozen, BOOT_ANY }, + { "map_hash_matches_libbpf", map_hash_matches_libbpf, BOOT_ANY }, + { "map_hash_multi_element", map_hash_multi_element, BOOT_ANY }, + { "map_hash_bad_size", map_hash_bad_size, BOOT_ANY }, + { "map_hash_unsupported_type", map_hash_unsupported_type, BOOT_ANY }, + { "lsm_signature_verdict", lsm_signature_verdict, BOOT_SEALED }, + { "signed_no_fd_array", signed_no_fd_array, BOOT_SEALED }, + { "signed_map_by_fd_rejected", signed_map_by_fd_rejected, BOOT_SEALED }, + { "signed_sparse_fd_array_rejected", signed_sparse_fd_array_rejected, BOOT_SEALED }, + { "bpf_keyring_provisioned", bpf_keyring_provisioned, BOOT_UNSEALED }, +}; + void test_signed_loader(void) { - if (test__start_subtest("loadtime_no_map")) - loadtime_no_map(); - if (test__start_subtest("loadtime_with_map")) - loadtime_with_map(); - if (test__start_subtest("metadata_match")) - metadata_match(); - if (test__start_subtest("signature_enforced")) - signature_enforced(); - if (test__start_subtest("signed_nonexcl_fd_array_rejected")) - signed_nonexcl_fd_array_rejected(); - if (test__start_subtest("signed_unfrozen_fd_array_rejected")) - signed_unfrozen_fd_array_rejected(); - if (test__start_subtest("signed_nonarray_fd_array_rejected")) - signed_nonarray_fd_array_rejected(); - if (test__start_subtest("signed_btf_fd_array_rejected")) - signed_btf_fd_array_rejected(); - if (test__start_subtest("signed_module_kfunc_rejected")) - signed_module_kfunc_rejected(); - if (test__start_subtest("signature_failure_logs")) - signature_failure_logs(); - if (test__start_subtest("signature_too_large")) - signature_too_large(); - if (test__start_subtest("signature_zero_size")) - signature_zero_size(); - if (test__start_subtest("signature_bad_keyring")) - signature_bad_keyring(); - if (test__start_subtest("metadata_ctx_max_entries_ignored")) - metadata_ctx_max_entries_ignored(); - if (test__start_subtest("metadata_ctx_initial_value_ignored")) - metadata_ctx_initial_value_ignored(); - if (test__start_subtest("signature_authenticates_insns")) - signature_authenticates_insns(); - if (test__start_subtest("signature_authenticates_metadata")) - signature_authenticates_metadata(); - if (test__start_subtest("hash_requires_frozen")) - hash_requires_frozen(); - if (test__start_subtest("no_update_after_freeze")) - no_update_after_freeze(); - if (test__start_subtest("freeze_writable_mmap")) - freeze_writable_mmap(); - if (test__start_subtest("no_writable_mmap_frozen")) - no_writable_mmap_frozen(); - if (test__start_subtest("map_hash_matches_libbpf")) - map_hash_matches_libbpf(); - if (test__start_subtest("map_hash_multi_element")) - map_hash_multi_element(); - if (test__start_subtest("map_hash_bad_size")) - map_hash_bad_size(); - if (test__start_subtest("map_hash_unsupported_type")) - map_hash_unsupported_type(); - if (test__start_subtest("lsm_signature_verdict")) - lsm_signature_verdict(); - if (test__start_subtest("signed_no_fd_array")) - signed_no_fd_array(); - if (test__start_subtest("signed_map_by_fd_rejected")) - signed_map_by_fd_rejected(); - if (test__start_subtest("signed_sparse_fd_array_rejected")) - signed_sparse_fd_array_rejected(); + bool unsealed = keyring_unsealed_boot(); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(subtests); i++) { + if (!test__start_subtest(subtests[i].name)) + continue; + if (subtests[i].boot == BOOT_SEALED && unsealed) { + printf("%s:SKIP:needs a boot without bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + if (subtests[i].boot == BOOT_UNSEALED && !unsealed) { + printf("%s:SKIP:needs bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + subtests[i].fn(); + } }
diff --git a/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c b/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c new file mode 100644 index 0000000..919f24d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c
@@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <network_helpers.h> +#include "skb_csum_complete.skel.h" + +static void run(struct bpf_program *prog, int expected_err) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + .flags = BPF_F_TEST_SKB_CHECKSUM_COMPLETE, + ); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &topts); + if (!ASSERT_EQ(err, expected_err, "test_run")) + return; + if (!expected_err) + ASSERT_EQ(topts.retval, 0, "retval"); +} + +void test_skb_csum_complete(void) +{ + struct skb_csum_complete *skel; + + skel = skb_csum_complete__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + if (test__start_subtest("change_tail_trim")) + run(skel->progs.change_tail_trim, 0); + if (test__start_subtest("change_tail_grow_write")) + run(skel->progs.change_tail_grow_write, 0); + if (test__start_subtest("store_no_recompute")) + run(skel->progs.store_no_recompute, -EBADMSG); + + skb_csum_complete__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf.c b/tools/testing/selftests/bpf/prog_tests/snprintf.c index 4e4a82d..ccfb02b 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf.c
@@ -15,8 +15,8 @@ #define EXP_SYM_OUT "schedule schedule+0x0/" #define MIN_SYM_RET sizeof(EXP_SYM_OUT) -/* The third specifier, %p, is a hashed pointer which changes on every reboot */ -#define EXP_ADDR_OUT "0000000000000000 ffff00000add4e55 " +/* The second specifier, %p, is a hashed pointer which changes on every reboot */ +#define EXP_ADDR_OUT "ffff00000add4e55 " #define EXP_ADDR_RET sizeof(EXP_ADDR_OUT "unknownhashedptr") #define EXP_STR_OUT "str1 a b c d e longstr"
diff --git a/tools/testing/selftests/bpf/prog_tests/socket_helpers.h b/tools/testing/selftests/bpf/prog_tests/socket_helpers.h index 0d59503..4a57099 100644 --- a/tools/testing/selftests/bpf/prog_tests/socket_helpers.h +++ b/tools/testing/selftests/bpf/prog_tests/socket_helpers.h
@@ -470,4 +470,16 @@ static inline const char *socket_kind_to_str(int sock_fd) return "???"; } +static inline const char *inet_family_str(sa_family_t family) +{ + switch (family) { + case AF_INET: + return "IPv4"; + case AF_INET6: + return "IPv6"; + default: + return "unknown"; + } +} + #endif // __SOCKET_HELPERS__
diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index 1c96a3c..235e1d5 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c
@@ -1232,22 +1232,6 @@ static void test_ops_cleanup(const struct bpf_map *map) } } -static const char *family_str(sa_family_t family) -{ - switch (family) { - case AF_INET: - return "IPv4"; - case AF_INET6: - return "IPv6"; - case AF_UNIX: - return "Unix"; - case AF_VSOCK: - return "VSOCK"; - default: - return "unknown"; - } -} - static const char *map_type_str(const struct bpf_map *map) { int type; @@ -1315,7 +1299,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, char s[MAX_TEST_NAME]; int map_fd; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); sotype_name = sotype_str(sotype); map_fd = bpf_map__fd(map); @@ -1355,7 +1339,7 @@ static void test_redir(struct test_sockmap_listen *skel, struct bpf_map *map, const struct redir_test *t; char s[MAX_TEST_NAME]; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { @@ -1387,7 +1371,7 @@ static void test_reuseport(struct test_sockmap_listen *skel, const struct reuseport_test *t; char s[MAX_TEST_NAME]; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); sotype_name = sotype_str(sotype);
diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index 6c96f2d..7dfdc4c 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c
@@ -955,7 +955,7 @@ static struct sockopt_test { /* ==================== prog_type ==================== */ { - .descr = "can attach only BPF_CGROUP_SETSOCKOP", + .descr = "can attach only BPF_CGROUP_SETSOCKOPT", .insns = { /* return 1 */ BPF_MOV64_IMM(BPF_REG_0, 1), @@ -969,7 +969,7 @@ static struct sockopt_test { }, { - .descr = "can attach only BPF_CGROUP_GETSOCKOP", + .descr = "can attach only BPF_CGROUP_GETSOCKOPT", .insns = { /* return 1 */ BPF_MOV64_IMM(BPF_REG_0, 1),
diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index e4e9374..ae400dc 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c
@@ -1,11 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include <test_progs.h> +#include <linux/perf_event.h> +#include <poll.h> +#include <sys/epoll.h> #include <sys/mman.h> +#include <sys/syscall.h> #include "stream.skel.h" #include "stream_fail.skel.h" +#define NMI_TIMEOUT_NS (5ULL * 1000 * 1000 * 1000) + void test_stream_failure(void) { RUN_TESTS(stream_fail); @@ -46,6 +52,10 @@ void test_stream_syscall(void) ret = -errno; ASSERT_EQ(ret, -EFAULT, "bad stream buf"); + ASSERT_LT(bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, UINT_MAX, NULL), 0, "error"); + ret = -errno; + ASSERT_EQ(ret, -EINVAL, "large stream buf"); + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); ASSERT_EQ(ret, 2, "bytes"); ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); @@ -58,6 +68,549 @@ void test_stream_syscall(void) stream__destroy(skel); } +static bool stream_fd_trigger(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; + + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(ret, "test_run") && ASSERT_OK(opts.retval, "retval"); +} + +static void test_stream_fd_open(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts); + struct stream *skel; + int fd, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + fd = bpf_prog_stream_open(0, BPF_STREAM_STDOUT, NULL); + ASSERT_EQ(fd, -EINVAL, "bad_prog_fd"); + + fd = bpf_prog_stream_open(prog_fd, 0, NULL); + ASSERT_EQ(fd, -ENOENT, "bad_stream_id"); + + opts.flags = BPF_F_RDONLY; + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDOUT, &opts); + ASSERT_EQ(fd, -EINVAL, "access_flag"); + + opts.flags = 1U << 31; + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDOUT, &opts); + ASSERT_EQ(fd, -EINVAL, "unknown_flag"); + + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDERR, NULL); + if (ASSERT_OK_FD(fd, "stderr")) + close(fd); + + stream__destroy(skel); +} + +static void test_stream_fd_nonblock(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4] = {}; + int fd, flags, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + + flags = fcntl(fd, F_GETFD); + ASSERT_GE(flags, 0, "getfd"); + ASSERT_NEQ(flags & FD_CLOEXEC, 0, "cloexec"); + flags = fcntl(fd, F_GETFL); + ASSERT_GE(flags, 0, "getfl"); + ASSERT_EQ(flags & O_ACCMODE, O_RDONLY, "readonly"); + ASSERT_NEQ(flags & O_NONBLOCK, 0, "nonblock"); + + ret = write(fd, "x", 1); + ASSERT_EQ(ret, -1, "write"); + ASSERT_EQ(errno, EBADF, "write_errno"); + ret = lseek(fd, 0, SEEK_SET); + ASSERT_EQ(ret, -1, "lseek"); + ASSERT_EQ(errno, ESPIPE, "lseek_errno"); + + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 0, "poll_empty"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_empty"); + ASSERT_EQ(errno, EAGAIN, "read_empty_errno"); + + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_data"); + ASSERT_NEQ(pfd.revents & POLLIN, 0, "pollin"); + ASSERT_EQ(pfd.revents & POLLHUP, 0, "no_pollhup"); + + ret = read(fd, buf, 2); + ASSERT_EQ(ret, 2, "read_first"); + ASSERT_OK(memcmp(buf, "fo", 2), "read_first_data"); + pfd.revents = 0; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_partial"); + ASSERT_NEQ(pfd.revents & POLLIN, 0, "pollin_partial"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 1, "read_rest"); + ASSERT_EQ(buf[0], 'o', "read_rest_data"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_drained"); + ASSERT_EQ(errno, EAGAIN, "read_drained_errno"); + +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_empty(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4]; + int fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_empty), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + + /* Empty output produces neither data nor readiness. */ + if (!stream_fd_trigger(skel->progs.stream_empty)) + goto out_close; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 0, "poll_empty_write"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_empty_write"); + ASSERT_EQ(errno, EAGAIN, "read_empty_write_errno"); + +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +struct stream_fd_read_ctx { + int fd; + ssize_t ret; + char buf[4]; +}; + +static void *stream_fd_read_thread(void *arg) +{ + struct stream_fd_read_ctx *ctx = arg; + + ctx->ret = read(ctx->fd, ctx->buf, sizeof(ctx->buf)); + return NULL; +} + +static int stream_fd_timed_join(pthread_t thread) +{ + struct timespec timeout; + + clock_gettime(CLOCK_REALTIME, &timeout); + timeout.tv_sec += 5; + return pthread_timedjoin_np(thread, NULL, &timeout); +} + +static void test_stream_fd_blocking(void) +{ + struct stream_fd_read_ctx ctx = {}; + struct stream *skel; + pthread_t thread; + bool thread_live = false; + int fd, flags, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, NULL); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + ctx.fd = fd; + + flags = fcntl(fd, F_GETFL); + ASSERT_GE(flags, 0, "getfl"); + ASSERT_EQ(flags & O_NONBLOCK, 0, "blocking"); + + ret = pthread_create(&thread, NULL, stream_fd_read_thread, &ctx); + if (!ASSERT_OK(ret, "pthread_create")) + goto out_close; + thread_live = true; + + usleep(50000); + ret = pthread_tryjoin_np(thread, NULL); + if (!ASSERT_EQ(ret, EBUSY, "read_blocks")) { + thread_live = ret != 0; + goto out_thread; + } + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_thread; + + ret = stream_fd_timed_join(thread); + if (!ASSERT_OK(ret, "pthread_join")) + goto out_thread; + thread_live = false; + ASSERT_EQ(ctx.ret, 3, "read_len"); + ASSERT_OK(memcmp(ctx.buf, "foo", 3), "read_data"); + + memset(&ctx, 0, sizeof(ctx)); + ctx.fd = fd; + ret = pthread_create(&thread, NULL, stream_fd_read_thread, &ctx); + if (!ASSERT_OK(ret, "pthread_create_eof")) + goto out_close; + thread_live = true; + + usleep(50000); + ret = pthread_tryjoin_np(thread, NULL); + if (!ASSERT_EQ(ret, EBUSY, "read_eof_blocks")) { + thread_live = ret != 0; + goto out_thread; + } + + stream__destroy(skel); + skel = NULL; + ret = stream_fd_timed_join(thread); + if (!ASSERT_OK(ret, "pthread_join_eof")) + goto out_thread; + thread_live = false; + ASSERT_EQ(ctx.ret, 0, "read_eof"); + +out_thread: + if (thread_live) { + stream__destroy(skel); + skel = NULL; + ret = stream_fd_timed_join(thread); + if (ret) { + pthread_cancel(thread); + pthread_join(thread, NULL); + } + } +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_hup(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct epoll_event event = { + .events = EPOLLIN | EPOLLET, + }; + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4] = {}; + int epfd = -1, fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out_close; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out_close; + + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_data")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin"); + ASSERT_EQ(event.events & EPOLLHUP, 0, "no_epollhup"); + + stream__destroy(skel); + skel = NULL; + event.events = 0; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_hup")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_with_hup"); + ASSERT_NEQ(event.events & EPOLLHUP, 0, "epollhup"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 3, "read_buffered"); + ASSERT_OK(memcmp(buf, "foo", 3), "read_buffered_data"); + pfd.revents = 0; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_drained_hup"); + ASSERT_EQ(pfd.revents & POLLIN, 0, "no_pollin_after_drain"); + ASSERT_NEQ(pfd.revents & POLLHUP, 0, "pollhup_after_drain"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 0, "read_eof"); + +out_close: + if (epfd >= 0) + close(epfd); + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_edge(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct epoll_event event = { + .events = EPOLLIN | EPOLLET, + }; + struct stream *skel; + char buf[8] = {}; + int epfd = -1, fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out_close; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out_close; + + /* Output into an empty stream raises an edge. */ + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_first")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_first"); + + /* + * Leave data behind, add more, then drain everything. Whether output + * into a non-empty stream raises an edge of its own is not part of the + * contract, so discard any pending event before re-arming the stream. + */ + ret = read(fd, buf, 2); + ASSERT_EQ(ret, 2, "read_partial"); + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 4, "read_rest"); + ASSERT_OK(memcmp(buf, "ofoo", 4), "read_rest_data"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_drained"); + ASSERT_EQ(errno, EAGAIN, "read_drained_errno"); + epoll_wait(epfd, &event, 1, 0); + + /* Output into the drained stream raises an edge again. */ + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_after_drain")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_after_drain"); + +out_close: + if (epfd >= 0) + close(epfd); + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_nmi_epoll(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .freq = 1, + .sample_freq = 10, + }; + struct epoll_event event = { + .events = EPOLLIN, + }; + struct bpf_link *link = NULL; + struct stream *skel; + __u64 deadline; + char buf[4] = {}; + int epfd = -1, fd = -1, pmu_fd = -1; + int ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_nmi), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out; + + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out; + + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, + PERF_FLAG_FD_CLOEXEC); + if (pmu_fd < 0 && (errno == ENOENT || errno == EOPNOTSUPP)) { + printf("%s:SKIP:no PERF_COUNT_HW_CPU_CYCLES\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_GE(pmu_fd, 0, "perf_event_open")) + goto out; + + link = bpf_program__attach_perf_event(skel->progs.stream_nmi, pmu_fd); + if (!ASSERT_OK_PTR(link, "attach_perf_event")) { + link = NULL; + goto out; + } + pmu_fd = -1; + + deadline = get_time_ns() + NMI_TIMEOUT_NS; + do { + ret = epoll_wait(epfd, &event, 1, 0); + } while (ret == 0 && get_time_ns() < deadline); + if (!ASSERT_EQ(ret, 1, "epoll_wait")) + goto out; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 3, "read_len"); + ASSERT_OK(memcmp(buf, "nmi", 3), "read_data"); + +out: + bpf_link__destroy(link); + if (pmu_fd >= 0) + close(pmu_fd); + if (epfd >= 0) + close(epfd); + if (fd >= 0) + close(fd); + stream__destroy(skel); +} + +void test_stream_fd(void) +{ + if (test__start_subtest("open")) + test_stream_fd_open(); + if (test__start_subtest("nonblock")) + test_stream_fd_nonblock(); + if (test__start_subtest("empty")) + test_stream_fd_empty(); + if (test__start_subtest("blocking")) + test_stream_fd_blocking(); + if (test__start_subtest("hup")) + test_stream_fd_hup(); + if (test__start_subtest("edge")) + test_stream_fd_edge(); + if (test__start_subtest("nmi_epoll")) + test_stream_fd_nmi_epoll(); +} + +void test_stream_oversize(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_oversize); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "oversize run"); + ASSERT_OK(opts.retval, "oversize retval"); + + stream__destroy(skel); +} + +void test_stream_partial_read(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + long page_size; + char *page, *buf; + char rest[8] = {}; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "ret"); + ASSERT_OK(opts.retval, "retval"); + + page_size = sysconf(_SC_PAGESIZE); + page = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(page, MAP_FAILED, "mmap")) { + stream__destroy(skel); + return; + } + /* Leave only the first page mapped so a straddling copy faults. */ + if (!ASSERT_OK(munmap(page + page_size, page_size), "munmap second page")) { + munmap(page, page_size * 2); + stream__destroy(skel); + return; + } + + buf = page + page_size - 1; + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 3, NULL); + ASSERT_EQ(ret, 1, "partial bytes"); + ASSERT_EQ(buf[0], 'f', "first byte"); + + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, rest, sizeof(rest), NULL); + ASSERT_EQ(ret, 2, "remaining bytes"); + ASSERT_OK(memcmp(rest, "oo", 2), "remaining data"); + + munmap(page, page_size); + stream__destroy(skel); +} + static void test_address(struct bpf_program *prog, unsigned long *fault_addr_p) { LIBBPF_OPTS(bpf_test_run_opts, opts);
diff --git a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c index 98db9ba..2b3ec2b 100644 --- a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c
@@ -4,6 +4,7 @@ #include "struct_ops_private_stack.skel.h" #include "struct_ops_private_stack_fail.skel.h" #include "struct_ops_private_stack_recur.skel.h" +#include "struct_ops_private_stack_large.skel.h" #if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) static void test_private_stack(void) @@ -78,6 +79,34 @@ static void test_private_stack_recur(void) struct_ops_private_stack_recur__destroy(skel); } +/* Two frames of 2 KiB each on the private stack */ +static void test_private_stack_large(void) +{ + struct struct_ops_private_stack_large *skel; + struct bpf_link *link; + + if (!is_large_stack_supported()) { + test__skip(); + return; + } + + skel = struct_ops_private_stack_large__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_large__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_1); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->val, 100 + 30 + 12, "val"); + + bpf_link__destroy(link); +cleanup: + struct_ops_private_stack_large__destroy(skel); +} + static void __test_struct_ops_private_stack(void) { if (test__start_subtest("private_stack")) @@ -86,6 +115,8 @@ static void __test_struct_ops_private_stack(void) test_private_stack_fail(); if (test__start_subtest("private_stack_recur")) test_private_stack_recur(); + if (test__start_subtest("private_stack_large")) + test_private_stack_large(); } #else static void __test_struct_ops_private_stack(void)
diff --git a/tools/testing/selftests/bpf/prog_tests/summarization.c b/tools/testing/selftests/bpf/prog_tests/summarization.c index 6951786..3e4bcf0 100644 --- a/tools/testing/selftests/bpf/prog_tests/summarization.c +++ b/tools/testing/selftests/bpf/prog_tests/summarization.c
@@ -53,17 +53,6 @@ static void test_aux(const char *main_prog_name, err = summarization_freplace__load(freplace); print_verifier_log(log); - /* The might_sleep extension doesn't work yet as sleepable calls are not - * allowed, but preserve the check in case it's supported later and then - * this particular combination can be enabled. - */ - if (!strcmp("might_sleep", replacement) && err) { - ASSERT_HAS_SUBSTR(log, "sleepable helper bpf_copy_from_user#", "error log"); - ASSERT_EQ(err, -EINVAL, "err"); - test__skip(); - goto out; - } - if (expect_load) { ASSERT_OK(err, "summarization_freplace__load"); } else { @@ -86,9 +75,8 @@ static void test_aux(const char *main_prog_name, * Also test a case when main subprogram itself is replaced and is a single * subprogram in a program. * - * This holds for might_sleep programs. It is ok to replace might_sleep with - * might_sleep and with does_not_sleep, but does_not_sleep cannot be replaced - * with might_sleep. + * An extension is verified as a non-sleepable program, hence nothing that runs + * sleepable can be replaced. */ void test_summarization_freplace(void) { @@ -111,7 +99,7 @@ void test_summarization_freplace(void) }, }; const char *pkt_err = "Extension program changes packet data"; - const char *slp_err = "Extension program may sleep"; + const char *slp_err = "Cannot extend a function that runs without RCU read lock"; struct { const char *func; bool has_side_effect; @@ -136,7 +124,8 @@ void test_summarization_freplace(void) if (!test__start_subtest(buf)) continue; test_aux(mains[t][i].main, mains[t][i].to_be_replaced, replacements[t][j].func, - mains[t][i].has_side_effect || !replacements[t][j].has_side_effect, + t == 0 && + (mains[t][i].has_side_effect || !replacements[t][j].has_side_effect), replacements[t][j].err_msg); } }
diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index c5c9d6c..88d2231 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c
@@ -6,9 +6,11 @@ #include "tailcall_bpf2bpf_hierarchy2.skel.h" #include "tailcall_bpf2bpf_hierarchy3.skel.h" #include "tailcall_freplace.skel.h" +#include "tailcall_freplace_multi.skel.h" #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" #include "tailcall_cgrp_storage_owner.skel.h" +#include "tailcall_large_stack.skel.h" #include "tailcall_cgrp_storage_no_storage.skel.h" #include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" @@ -1655,6 +1657,77 @@ static void test_tailcall_bpf2bpf_freplace(void) tc_bpf2bpf__destroy(tc_skel); } +static void test_tailcall_freplace_multi(void) +{ + struct tailcall_freplace_multi *freplace_skel = NULL; + struct bpf_link *link_subprog = NULL, *link_entry = NULL; + struct tc_bpf2bpf *tc_skel = NULL; + int tc_prog_fd, map_fd, key = 0, err; + + tc_skel = tc_bpf2bpf__open_and_load(); + if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load")) + return; + + tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc); + + freplace_skel = tailcall_freplace_multi__open(); + if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace_multi__open")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK(err, "set_attach_target subprog_tc")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK(err, "set_attach_target entry_tc")) + goto out; + + err = tailcall_freplace_multi__load(freplace_skel); + if (!ASSERT_OK(err, "tailcall_freplace_multi__load")) + goto out; + + map_fd = bpf_map__fd(freplace_skel->maps.jmp_table); + + link_subprog = bpf_program__attach_freplace(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK_PTR(link_subprog, "attach_freplace subprog_tc")) + goto out; + + link_entry = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK_PTR(link_entry, "attach_freplace entry_tc")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with extended prog")) + goto out; + + err = bpf_link__destroy(link_entry); + link_entry = NULL; + if (!ASSERT_OK(err, "destroy entry link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with still extended prog")) + goto out; + + err = bpf_link__destroy(link_subprog); + link_subprog = NULL; + if (!ASSERT_OK(err, "destroy subprog link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + ASSERT_OK(err, "update jmp_table"); + +out: + bpf_link__destroy(link_subprog); + bpf_link__destroy(link_entry); + tailcall_freplace_multi__destroy(freplace_skel); + tc_bpf2bpf__destroy(tc_skel); +} + static void test_tailcall_failure() { RUN_TESTS(tailcall_fail); @@ -1953,6 +2026,45 @@ static void test_tailcall_bpf2bpf_fexit_links(void) tailcall_bpf2bpf2__destroy(skel_tc); } +/* + * test_tailcall_large_stack runs a tail call made from a subprog with a 1536 + * byte frame, under a 240-byte caller, into a program with a 2 KiB frame: + * + * entry (240) --call-> subprog_tail (1536) --tailcall-> classifier_0 (2048) + */ +static void test_tailcall_large_stack(void) +{ + struct tailcall_large_stack *skel; + int err, prog_fd, map_fd, key = 0; + char buff[128] = {}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = buff, + .data_size_in = sizeof(buff), + .repeat = 1, + ); + + if (!is_large_stack_supported()) { + test__skip(); + return; + } + + skel = tailcall_large_stack__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tailcall_large_stack__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.classifier_0); + map_fd = bpf_map__fd(skel->maps.jmp_table); + err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update jmp_table")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.entry), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 42 + 7, "retval"); +out: + tailcall_large_stack__destroy(skel); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -2005,6 +2117,8 @@ void test_tailcalls(void) test_tailcall_freplace(); if (test__start_subtest("tailcall_bpf2bpf_freplace")) test_tailcall_bpf2bpf_freplace(); + if (test__start_subtest("tailcall_freplace_multi")) + test_tailcall_freplace_multi(); if (test__start_subtest("tailcall_failure")) test_tailcall_failure(); if (test__start_subtest("tailcall_sleepable")) @@ -2022,4 +2136,6 @@ void test_tailcalls(void) test_tailcall_callback(); if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) test_tailcall_bpf2bpf_fexit_links(); + if (test__start_subtest("tailcall_large_stack")) + test_tailcall_large_stack(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c b/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c new file mode 100644 index 0000000..337f9d3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c
@@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ +#include <sys/epoll.h> + +#include "test_progs.h" +#include "cgroup_helpers.h" +#include "network_helpers.h" + +#include "tcp_autolowat.skel.h" + +#define SOL_BPF 0xdeadbeef +#define BPF_TCP_AUTOLOWAT 0x8badf00d + +struct rpc_descriptor { + u32 header_len; + u32 payload_len; +}; + +enum rpc_event_type { + RPC_EVENT_END, + RPC_EVENT_AUTOLOWAT, + RPC_EVENT_SEND, + RPC_EVENT_RECV, + RPC_EVENT_EPOLL, + RPC_EVENT_RCVLOWAT, +}; + +struct rpc_event { + enum rpc_event_type type; + union { + int len; + int nfds; + int val; + int rcvlowat; + }; +}; + +#define RPC_DESC_SIZE (sizeof(struct rpc_descriptor)) + +static struct rpc_test_case { + char data[4096]; + struct rpc_descriptor desc[32]; + struct rpc_event event[32]; +} rpc_test_cases[] = { + { + .desc = { + { .header_len = 100, .payload_len = 150 }, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Single full RPC message in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE + 100 + 150}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 100 + 150}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Two full RPC messages in skb. */ + {.type = RPC_EVENT_SEND, .len = (RPC_DESC_SIZE + 100 + 150) * 2}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + /* Single full RPC message in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE + 100 + 150}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 3}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Two full RPC messages in skb. */ + {.type = RPC_EVENT_SEND, .len = (RPC_DESC_SIZE + 100 + 150) * 2}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + /* Only the next descriptor in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 200, .payload_len = 500}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* The first descriptor is partial. */ + {.type = RPC_EVENT_SEND, .len = 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE}, + /* The first descriptor is available. */ + {.type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE - 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* The first header is ready. */ + {.type = RPC_EVENT_SEND, .len = 100}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* skb has the first payload and 1 byte of the next descriptor. */ + {.type = RPC_EVENT_SEND, .len = 150 + 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* After reading the first RPC message, SO_RCVLOWAT should be RPC_DESC_SIZE. */ + {.type = RPC_EVENT_RECV, .len = RPC_DESC_SIZE + 150 + 100}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE}, + /* The second descriptor is available. */ + {.type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE - 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 200 + 500}, + }, + }, +}; + +struct tcp_autolowat_test_cb { + int saved_netns; + union { + int fd[4]; + struct { + int server, client, child; + int epoll; + }; + }; +}; + +static void tcp_autolowat_teardown_cb(struct tcp_autolowat_test_cb *cb) +{ + int i, err; + + for (i = 0; i < ARRAY_SIZE(cb->fd); i++) { + if (cb->fd[i] != -1) + close(cb->fd[i]); + } + + if (cb->saved_netns != -1) { + err = setns(cb->saved_netns, CLONE_NEWNET); + ASSERT_OK(err, "restore netns"); + + close(cb->saved_netns); + } +} + +static int tcp_autolowat_setup_cb(struct tcp_autolowat_test_cb *cb, int family) +{ + struct epoll_event ev = {}; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(cb->fd); i++) + cb->fd[i] = -1; + + cb->saved_netns = open("/proc/self/ns/net", O_RDONLY); + if (!ASSERT_OK_FD(cb->saved_netns, "save netns")) + goto err; + + err = unshare(CLONE_NEWNET); + if (!ASSERT_OK(err, "unshare")) + goto err; + + err = system("ip link set dev lo up"); + if (!ASSERT_OK(err, "set up lo")) + goto err; + + cb->server = start_server(family, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_OK_FD(cb->server, "start_server")) + goto err; + + cb->client = connect_to_fd(cb->server, 0); + if (!ASSERT_OK_FD(cb->client, "connect_to_fd")) + goto err; + + cb->child = accept(cb->server, NULL, NULL); + if (!ASSERT_OK_FD(cb->child, "accept")) + goto err; + + cb->epoll = epoll_create1(0); + if (!ASSERT_OK_FD(cb->epoll, "epoll_create")) + goto err; + + ev.events = EPOLLIN; + ev.data.fd = cb->child; + + err = epoll_ctl(cb->epoll, EPOLL_CTL_ADD, cb->child, &ev); + if (!ASSERT_OK(err, "epoll_ctl")) + goto err; + + return 0; + +err: + tcp_autolowat_teardown_cb(cb); + return -1; +} + +static int tcp_autolowat_build_data(struct rpc_test_case *test_case) +{ + struct rpc_descriptor *desc = test_case->desc; + char *ptr = test_case->data; + int rpc_size; + + memset(ptr, 0, sizeof(test_case->data)); + + while (desc->header_len + desc->payload_len) { + rpc_size = sizeof(*desc) + desc->header_len + desc->payload_len; + + if (!ASSERT_LE(ptr + rpc_size - test_case->data, + sizeof(test_case->data), "data overflow")) + return 1; + + memcpy(ptr, desc, sizeof(*desc)); + ptr += rpc_size; + desc++; + } + + if (!ASSERT_GT(ptr - test_case->data, 0, "no data")) + return 1; + + return 0; +} + +static void tcp_autolowat_run_rpc_test(struct tcp_autolowat_test_cb *cb, + struct rpc_test_case *test_case) +{ + struct rpc_event *event = test_case->event; + char *ptr = test_case->data; + struct epoll_event ev; + socklen_t optlen; + int err, optval; + char buf[4096]; + + if (tcp_autolowat_build_data(test_case)) + return; + + while (1) { + switch (event->type) { + case RPC_EVENT_END: + return; + case RPC_EVENT_AUTOLOWAT: + err = setsockopt(cb->child, SOL_BPF, BPF_TCP_AUTOLOWAT, + &event->val, sizeof(event->val)); + if (!ASSERT_OK(err, "setsockopt")) + return; + break; + case RPC_EVENT_SEND: + err = send(cb->client, ptr, event->len, 0); + if (!ASSERT_EQ(err, event->len, "send")) + return; + + ptr += event->len; + break; + case RPC_EVENT_RECV: + err = recv(cb->child, buf, event->len, 0); + if (!ASSERT_EQ(err, event->len, "recv")) + return; + break; + case RPC_EVENT_EPOLL: + err = epoll_wait(cb->epoll, &ev, 1, 100); + if (!ASSERT_EQ(err, event->nfds, "epoll_wait")) + return; + break; + case RPC_EVENT_RCVLOWAT: + optval = 0; + optlen = sizeof(optval); + + err = getsockopt(cb->child, SOL_SOCKET, SO_RCVLOWAT, &optval, &optlen); + if (!ASSERT_OK(err, "getsockopt") || + !ASSERT_EQ(optval, event->rcvlowat, "rcvlowat")) + return; + break; + } + + event++; + } +} + +static void tcp_autolowat_run_rpc_tests(struct tcp_autolowat *skel, int family) +{ + struct tcp_autolowat_test_cb cb; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(rpc_test_cases); i++) { + memset(skel->bss->test_name, 0, sizeof(skel->bss->test_name)); + + snprintf(skel->bss->test_name, sizeof(skel->bss->test_name), + "AF_INET%c rpc_test_cases[%d]", + family == AF_INET ? ' ' : '6', i); + + if (!test__start_subtest(skel->bss->test_name)) + continue; + + err = tcp_autolowat_setup_cb(&cb, family); + if (err) + continue; + + tcp_autolowat_run_rpc_test(&cb, &rpc_test_cases[i]); + tcp_autolowat_teardown_cb(&cb); + } +} + +static void tcp_autolowat_run_tests(struct tcp_autolowat *skel) +{ + tcp_autolowat_run_rpc_tests(skel, AF_INET); + tcp_autolowat_run_rpc_tests(skel, AF_INET6); +} + +void test_tcp_autolowat(void) +{ + struct tcp_autolowat *skel; + struct bpf_link *link[2]; + int cgroup; + + skel = tcp_autolowat__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + cgroup = test__join_cgroup("/tcp_autolowat"); + if (!ASSERT_GE(cgroup, 0, "join_cgroup")) + goto destroy_skel; + + link[0] = bpf_map__attach_cgroup_opts(skel->maps.tcp_autolowat_ops, cgroup, NULL); + if (!ASSERT_OK_PTR(link[0], "attach_cgroup(tcp_autolowat_ops)")) + goto close_cgroup; + + link[1] = bpf_program__attach_cgroup(skel->progs.tcp_autolowat_setsockopt, cgroup); + if (!ASSERT_OK_PTR(link[1], "attach_cgroup(SETSOCKOPT)")) + goto destroy_sockops; + + tcp_autolowat_run_tests(skel); + + bpf_link__destroy(link[1]); +destroy_sockops: + bpf_link__destroy(link[0]); +close_cgroup: + close(cgroup); +destroy_skel: + tcp_autolowat__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c index 7f9f54b..e7580b9 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c
@@ -5,6 +5,15 @@ #include "struct_ops_arena.skel.h" #include "struct_ops_arena_attach.skel.h" #include "struct_ops_arena_fail.skel.h" +#include "struct_ops_arena_member.skel.h" +#include "struct_ops_arena_member_render.skel.h" +#include "struct_ops_arena_member_fail__union.skel.h" +#include "struct_ops_arena_member_fail__xchg.skel.h" +#include "struct_ops_arena_member_fail__mismatch.skel.h" +#include "struct_ops_arena_member_fail__mismatch_mem.skel.h" +#include "struct_ops_arena_member_fail__write.skel.h" +#include "struct_ops_arena_member_fail__no_arena.skel.h" +#include "struct_ops_arena_autobind.skel.h" #if defined(__x86_64__) || defined(__aarch64__) /* @@ -101,6 +110,167 @@ static void arena_arg_attach(void) struct_ops_arena__destroy(skel); } + +/* Attach @st_ops and run @trigger once, it returns 0 when everything checked out. */ +static void attach_and_trigger(struct bpf_map *st_ops, struct bpf_program *trigger) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_link *link; + int err; + + link = bpf_map__attach_struct_ops(st_ops); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +} + +/* + * Attach a callback that reads arena pointers out of its argument struct. + * The header is shared with the module, so the members are arena pointers + * on the program side as well. + */ +static void arena_member(void) +{ + struct struct_ops_arena_member *skel; + + skel = struct_ops_arena_member__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member, skel->progs.trigger); + struct_ops_arena_member__destroy(skel); +} + +/* @prog_fd lists exactly one map, the one with @map_id. */ +static void assert_prog_map(int prog_fd, __u32 map_id, const char *name) +{ + __u32 map_ids[2] = {}, len = sizeof(struct bpf_prog_info); + struct bpf_prog_info info = { + .nr_map_ids = ARRAY_SIZE(map_ids), + .map_ids = ptr_to_u64(map_ids), + }; + int err; + + err = bpf_prog_get_info_by_fd(prog_fd, &info, &len); + if (!ASSERT_OK(err, name)) + return; + if (!ASSERT_EQ(info.nr_map_ids, 1, name)) + return; + ASSERT_EQ(map_ids[0], map_id, name); +} + +/* + * libbpf binds the arena of an object to every program in it: a callback + * with an __arena argument loads without referring to the arena, and an + * unrelated program gets the arena as well. + */ +static void arena_autobind(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open_and_load")) + return; + + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.test_arena_cb), info.id, "test_arena_cb"); + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); + + attach_and_trigger(skel->maps.testmod_arena_autobind, skel->progs.trigger); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * The same binding through bpf_object__prepare() and bpf_program__clone(), the + * way veristat loads a program: the arena takes slot 0 of a continuous + * fd_array, next to the module BTF of the kfunc the trigger calls. + */ +static void arena_autobind_clone(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + struct bpf_program *prog; + int err, fd; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + bpf_object__for_each_program(prog, skel->obj) { + fd = bpf_program__clone(prog, NULL); + if (!ASSERT_GE(fd, 0, bpf_program__name(prog))) + continue; + assert_prog_map(fd, info.id, bpf_program__name(prog)); + close(fd); + } +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * An arena exclusive to a program is bound to that program all the same: the + * kernel checks an exclusive map against the digest of the final program, so + * the binding through the fd_array passes for the program the map is + * exclusive to. + */ +static void arena_autobind_excl(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + /* the other programs need the arena too, and cannot have it */ + bpf_map__set_autocreate(skel->maps.testmod_arena_autobind, false); + bpf_program__set_autoload(skel->progs.trigger, false); + err = bpf_map__set_exclusive_program(skel->maps.arena, skel->progs.unrelated); + if (!ASSERT_OK(err, "set_exclusive_program")) + goto out; + + err = struct_ops_arena_autobind__load(skel); + if (!ASSERT_OK(err, "load")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* The same through the vmlinux.h rendering of the members, which takes casts. */ +static void arena_member_render(void) +{ + struct struct_ops_arena_member_render *skel; + + skel = struct_ops_arena_member_render__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member_render__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member_render, skel->progs.trigger); + struct_ops_arena_member_render__destroy(skel); +} #endif /* @@ -116,12 +286,37 @@ void serial_test_struct_ops_arena(void) * arena arguments", so the programs cannot even load. */ #if defined(__x86_64__) || defined(__aarch64__) + int has_tag; + if (test__start_subtest("arena_arg")) arena_arg(); if (test__start_subtest("arena_arg_fail")) arena_arg_fail(); if (test__start_subtest("arena_arg_attach")) arena_arg_attach(); + /* the tags reach the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_arena_member_args", "ptr", "arena"); + if (ASSERT_GE(has_tag, 0, "testmod_btf") && has_tag) { + if (test__start_subtest("arena_member")) + arena_member(); + if (test__start_subtest("arena_member_render")) + arena_member_render(); + RUN_TESTS(struct_ops_arena_member_fail__union); + RUN_TESTS(struct_ops_arena_member_fail__xchg); + RUN_TESTS(struct_ops_arena_member_fail__mismatch); + RUN_TESTS(struct_ops_arena_member_fail__mismatch_mem); + RUN_TESTS(struct_ops_arena_member_fail__write); + RUN_TESTS(struct_ops_arena_member_fail__no_arena); + } else if (test__start_subtest("arena_member")) { + /* so that the missing member tests show up as skipped */ + test__skip(); + } + if (test__start_subtest("arena_autobind")) + arena_autobind(); + if (test__start_subtest("arena_autobind_clone")) + arena_autobind_clone(); + if (test__start_subtest("arena_autobind_excl")) + arena_autobind_excl(); #else test__skip(); #endif
diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c index 0f321e8..8767e5f 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c
@@ -3,7 +3,62 @@ #include <test_progs.h> #include "struct_ops_multi_args.skel.h" +static void test_refcounted_multi(void) +{ + struct struct_ops_multi_args *skel; + char log_buf[4096] = {}; + int err; + + skel = struct_ops_multi_args__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_multi_args__open")) + return; + + bpf_program__set_log_buf(skel->progs.test_refcounted_multi, log_buf, sizeof(log_buf)); + + err = struct_ops_multi_args__load(skel); + if (!ASSERT_EQ(err, -EINVAL, "struct_ops_multi_args__load")) + goto out; + + ASSERT_HAS_SUBSTR(log_buf, "program with __ref argument cannot tail call", + "check_verifier_log_message"); + +out: + struct_ops_multi_args__destroy(skel); +} + +static void test_trampoline_stack_args(void) +{ + struct struct_ops_multi_args *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_multi_args__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_multi_args__open")) + return; + + bpf_program__set_autoload(skel->progs.test_refcounted_multi, false); + skel->struct_ops.testmod_ref_acquire->test_refcounted_multi = NULL; + + err = struct_ops_multi_args__load(skel); + if (!ASSERT_OK(err, "struct_ops_multi_args__load")) + goto out; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_ref_acquire); + if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) + goto out; + + ASSERT_EQ(skel->bss->trampoline_stack_arg9, 9999, "check_stack_passed_arg9"); + +out: + bpf_link__destroy(link); + struct_ops_multi_args__destroy(skel); +} + void test_struct_ops_multi_args(void) { - RUN_TESTS(struct_ops_multi_args); + if (test__start_subtest("test_refcounted_multi")) + test_refcounted_multi(); + + if (test__start_subtest("test_trampoline_stack_args")) + test_trampoline_stack_args(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c new file mode 100644 index 0000000..028f070 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c
@@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_nullable.skel.h" +#include "struct_ops_nullable_fail.skel.h" +#include "struct_ops_nullable_fail__percpu.skel.h" +#include "struct_ops_nullable_probed__plain.skel.h" +#include "struct_ops_nullable_probed__scalar.skel.h" +#include "struct_ops_nullable_probed__untrusted.skel.h" + +/* + * Attach a callback that gets a trusted and a trusted __nullable task pointer + * through its argument struct and drive it through the + * bpf_testmod_ops3_call_test_nullable() kfunc. + */ +static void nullable_member(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_nullable *skel; + struct bpf_link *link; + int err; + + skel = struct_ops_nullable__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_nullable__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_nullable); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +out: + struct_ops_nullable__destroy(skel); +} + +/* Serialized because it attaches the singleton bpf_testmod_ops3. */ +void serial_test_struct_ops_nullable(void) +{ + int has_tag; + + /* the tag reaches the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_nullable_args", "task_or_null", "nullable"); + if (!ASSERT_GE(has_tag, 0, "testmod_btf")) + return; + if (!has_tag) { + printf("%s:SKIP: no nullable type tag in the bpf_testmod BTF\n", __func__); + test__skip(); + return; + } + if (test__start_subtest("nullable_member")) + nullable_member(); + RUN_TESTS(struct_ops_nullable_fail); + RUN_TESTS(struct_ops_nullable_fail__percpu); + RUN_TESTS(struct_ops_nullable_probed__plain); + RUN_TESTS(struct_ops_nullable_probed__scalar); + RUN_TESTS(struct_ops_nullable_probed__untrusted); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_work.c b/tools/testing/selftests/bpf/prog_tests/test_task_work.c index 774b31a..ca42609 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_work.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_work.c
@@ -5,10 +5,15 @@ #include <stdio.h> #include "task_work.skel.h" #include "task_work_fail.skel.h" +#include "task_work_race.skel.h" +#include "task_work_busy.skel.h" #include <linux/bpf.h> #include <linux/perf_event.h> #include <sys/syscall.h> #include <time.h> +#include <pthread.h> +#include <sched.h> +#include <unistd.h> static int perf_event_open(__u32 type, __u64 config, int pid) { @@ -85,7 +90,7 @@ static void task_work_run(const char *prog_name, const char *map_name) skel = task_work__open(); if (!ASSERT_OK_PTR(skel, "task_work__open")) - return; + goto cleanup; bpf_object__for_each_program(prog, skel->obj) { bpf_program__set_autoload(prog, false); @@ -155,3 +160,692 @@ void test_task_work(void) RUN_TESTS(task_work_fail); } + +#define TASK_WORK_RACE_ROUNDS 1500 +#define TASK_WORK_POLL_US 100 +#define TASK_WORK_POLL_ITERS 20000 +#define TASK_WORK_START_POLL_ITERS 50000 + +/* Must match progs/task_work_race.c. */ +enum task_work_race_status { + RACE_ARM_SEQ, + RACE_READY_SEQ, + /* High 32 bits hold the sequence, low 32 bits the signed error. */ + RACE_RESULT, +}; + +struct task_work_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +struct task_work_race_ctx { + int stop; + int setup_err; + int trigger_tid; + int target_tid; +}; + +struct task_work_race_thread_arg { + struct task_work_race_ctx *ctx; + int cpu; + int *tid; +}; + +static int task_work_race_pin_cpu(int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +} + +static void *task_work_race_worker(void *arg) +{ + struct task_work_race_thread_arg *thread = arg; + struct task_work_race_ctx *ctx = thread->ctx; + int err; + + err = task_work_race_pin_cpu(thread->cpu); + if (err) + __atomic_store_n(&ctx->setup_err, err, __ATOMIC_RELEASE); + __atomic_store_n(thread->tid, syscall(__NR_gettid), __ATOMIC_RELEASE); + while (!__atomic_load_n(&ctx->stop, __ATOMIC_ACQUIRE)) + getppid(); + return NULL; +} + +static int task_work_status(struct bpf_map *map, int idx, __s64 *value) +{ + return bpf_map_lookup_elem(bpf_map__fd(map), &idx, value); +} + +static int task_work_set_status(struct bpf_map *map, int idx, __s64 value) +{ + return bpf_map_update_elem(bpf_map__fd(map), &idx, &value, BPF_ANY); +} + +/* Wait for seq, keeping polling errors separate from the scheduling result. */ +static int task_work_wait_seq(struct bpf_map *map, int idx, __u32 seq, + __s32 *result) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(map, idx, &value) && + (__u32)((__u64)value >> 32) == seq) { + *result = (__s32)value; + return 0; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_race_wait_ready(struct task_work_race *skel, __u32 seq) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, RACE_READY_SEQ, &value) && + value == seq) + return 0; + if (!task_work_status(skel->maps.status, RACE_RESULT, &value) && + (__u32)((__u64)value >> 32) == seq) + /* + * The BPF program finished between the two reads: + * RESULT makes waiting for READY unnecessary. Any error + * that occurred before READY is checked by the caller. + */ + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_race_prepare_elem(struct task_work_race *skel) +{ + struct task_work_value value = {}; + int key = 0; + + if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.hmap), &key, &value)) + return 0; + if (errno != ENOENT) + return -errno; + if (bpf_map_update_elem(bpf_map__fd(skel->maps.hmap), &key, &value, + BPF_NOEXIST)) + return -errno; + return 0; +} + +static int task_work_race_arm(struct task_work_race *skel, __u32 seq) +{ + if (task_work_set_status(skel->maps.status, RACE_ARM_SEQ, seq)) + return -errno; + return task_work_race_wait_ready(skel, seq); +} + +static int task_work_race_check_done(struct task_work_race *skel, __u32 seq) +{ + __s32 result; + int err; + + err = task_work_wait_seq(skel->maps.status, RACE_RESULT, seq, &result); + return err ?: result; +} + +static int task_work_race_wait_callback(struct task_work_race *skel, + __u32 seq) +{ + int i, key = seq; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __u64 completed = 0; + + if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.completed), + &key, &completed) && completed) + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +void serial_test_task_work_race(void) +{ + struct task_work_race_ctx ctx = {}; + struct task_work_race_thread_arg trigger_arg = { + .ctx = &ctx, + .tid = &ctx.trigger_tid, + }; + struct task_work_race_thread_arg target_arg = { + .ctx = &ctx, + .tid = &ctx.target_tid, + }; + struct task_work_race *skel; + pthread_t trigger, target; + cpu_set_t allowed; + int cpu, cpu_count = 0; + bool target_started = false, trigger_started = false; + int err, i, key = 0; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) { + ASSERT_OK(-errno, "sched_getaffinity"); + return; + } + for (cpu = 0; cpu < CPU_SETSIZE && cpu_count < 2; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + if (!cpu_count) + trigger_arg.cpu = cpu; + else + target_arg.cpu = cpu; + cpu_count++; + } + if (cpu_count < 2) { + printf("%s:SKIP:need two CPUs in the process affinity mask\n", + __func__); + test__skip(); + return; + } + + skel = task_work_race__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + if (!ASSERT_OK(task_work_race__attach(skel), "attach")) + goto cleanup; + + err = pthread_create(&target, NULL, task_work_race_worker, &target_arg); + if (!ASSERT_OK(err, "pthread_create target")) + goto cleanup; + target_started = true; + err = pthread_create(&trigger, NULL, task_work_race_worker, &trigger_arg); + if (!ASSERT_OK(err, "pthread_create trigger")) + goto stop; + trigger_started = true; + + for (i = 0; i < TASK_WORK_START_POLL_ITERS; i++) { + if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) + break; + usleep(TASK_WORK_POLL_US); + } + if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), + "thread start")) + goto stop; + if (!ASSERT_OK(__atomic_load_n(&ctx.setup_err, __ATOMIC_ACQUIRE), + "thread affinity")) + goto stop; + + skel->bss->trigger_tid = __atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE); + skel->bss->target_tid = __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE); + + for (i = 0; i < TASK_WORK_RACE_ROUNDS; i++) { + __u32 seq = i + 1; + int variant = i % 3; + + if (!ASSERT_OK(task_work_race_prepare_elem(skel), "prepare elem") || + !ASSERT_OK(task_work_race_arm(skel, seq), "round ready")) + goto stop; + + if (variant == 2 && + !ASSERT_OK(bpf_map_delete_elem(bpf_map__fd(skel->maps.hmap), + &key), "early delete")) + goto stop; + + err = task_work_race_check_done(skel, seq); + /* + * Variant 2 deletes the element after READY. A scheduling attempt + * racing with deletion can return -EBUSY, which the BPF program + * retries without publishing RESULT. The retry can then report + * -ENOENT because the element is gone; accept that terminal result. + * A later variant 0 round recreates the element. + */ + if (variant == 2 && err == -ENOENT) + err = 0; + if (!ASSERT_OK(err, "schedule")) { + fprintf(stderr, "round %d schedule failed: %d\n", i, err); + goto stop; + } + + if (variant != 2 && + !ASSERT_OK(task_work_race_wait_callback(skel, seq), + "callback")) + goto stop; + + if (variant == 1 && + !ASSERT_OK(bpf_map_delete_elem(bpf_map__fd(skel->maps.hmap), + &key), "late delete")) + goto stop; + } + + /* A distinct completion generation prevents an old callback satisfying this. */ + if (!ASSERT_OK(task_work_race_prepare_elem(skel), "final prepare") || + !ASSERT_OK(task_work_race_arm(skel, TASK_WORK_RACE_ROUNDS + 1), + "final ready") || + !ASSERT_OK(task_work_race_check_done(skel, + TASK_WORK_RACE_ROUNDS + 1), + "final schedule") || + !ASSERT_OK(task_work_race_wait_callback(skel, + TASK_WORK_RACE_ROUNDS + 1), + "final callback")) + goto stop; + +stop: + __atomic_store_n(&ctx.stop, 1, __ATOMIC_RELEASE); + if (trigger_started) + pthread_join(trigger, NULL); + if (target_started) + pthread_join(target, NULL); +cleanup: + task_work_race__destroy(skel); +} + +#define TASK_WORK_BUSY_ROUNDS 32 +#define TASK_WORK_BUSY_MAX_ATTEMPTS 96 + +/* Must match progs/task_work_busy.c. */ +enum task_work_busy_status { + BUSY_ARM_SEQ, + /* + * High 32 bits hold the sequence. BUSY_WINDOW low bits: 1 open, + * 2 the in-window attempt was seen before the handler returned, + * 3 the wait bound was reached without it. Result slots: signed + * error in the low 32 bits. + */ + BUSY_WINDOW, + BUSY_ACK, + BUSY_GEN1_RESULT, + BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, + BUSY_CB2_COUNT, + BUSY_CB2_SEQ, + BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + BUSY_STATUS_MAX, +}; + +enum task_work_busy_attempt { + TWB_VALID, + TWB_MISSED, + TWB_HARD_FAIL, +}; + +static const char *twb_attempt_name(enum task_work_busy_attempt r) +{ + switch (r) { + case TWB_VALID: + return "valid"; + case TWB_MISSED: + return "missed"; + default: + return "fail"; + } +} + +static int task_work_busy_wait_value(struct task_work_busy *skel, int idx, + __s64 want) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, idx, &value) && value == want) + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +/* Wait until the window slot reports a final state (2 or 3) for seq. */ +static int task_work_busy_wait_window(struct task_work_busy *skel, __u32 seq, + __s64 *window) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, BUSY_WINDOW, &value) && + (__u32)((__u64)value >> 32) == seq && + (__u32)value >= 2) { + *window = value; + return 0; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_busy_prepare_elem(struct task_work_busy *skel, int key) +{ + struct task_work_value value = {}; + + if (!bpf_map_update_elem(bpf_map__fd(skel->maps.hmap), &key, &value, + BPF_NOEXIST)) + return 0; + return -errno; +} + +static int task_work_busy_reset(struct task_work_busy *skel) +{ + static const int zeroed[] = { + BUSY_WINDOW, BUSY_ACK, BUSY_GEN1_RESULT, BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, BUSY_CB2_COUNT, BUSY_CB2_SEQ, BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(zeroed); i++) + if (task_work_set_status(skel->maps.status, zeroed[i], 0)) + return -errno; + return 0; +} + +/* + * join waits for clear_child_tid, which is cleared before exit_task_work(). + * PID lookup stops finding the worker only after its exit task_work has run. + * Probe with signal 0; no signal is delivered. Do not create new workers + * until both old TIDs have disappeared. + */ +static int task_work_busy_wait_exit(int tid) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + if (syscall(__NR_tgkill, getpid(), tid, 0)) { + if (errno == ESRCH) + return 0; + return -errno; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +/* + * Run one attempt on dedicated workers and report its outcome. The + * workers are stopped, joined and waited out of PID lookup before another + * attempt can reset the shared state. join alone does not wait for their + * exit task_work to finish. + */ +static enum task_work_busy_attempt +task_work_busy_run_attempt(struct task_work_busy *skel, int seq, + int trigger_cpu, int target_cpu, __s64 *out_wait_ns) +{ + struct task_work_race_ctx ctx = {}; + struct task_work_race_thread_arg trigger_arg = { + .ctx = &ctx, + .cpu = trigger_cpu, + .tid = &ctx.trigger_tid, + }; + struct task_work_race_thread_arg target_arg = { + .ctx = &ctx, + .cpu = target_cpu, + .tid = &ctx.target_tid, + }; + enum task_work_busy_attempt result = TWB_HARD_FAIL; + pthread_t trigger, target; + bool trigger_started = false, target_started = false; + bool cleanup_ok = true; + __s64 window = 0, wait_ns = 0, count = -1; + __s32 res = 0; + int err, i; + + *out_wait_ns = 0; + /* Do not let a reused TID match the previous attempt's filter. */ + skel->bss->trigger_tid = 0; + skel->bss->target_tid = 0; + + /* Clear old state before enabling workers that can read it. */ + if (!ASSERT_OK(task_work_busy_prepare_elem(skel, seq), "prepare elem")) + goto out; + if (!ASSERT_OK(task_work_busy_reset(skel), "reset slots")) + goto out; + + err = pthread_create(&target, NULL, task_work_race_worker, &target_arg); + if (err) { + ASSERT_OK(err, "pthread_create target"); + goto out; + } + target_started = true; + err = pthread_create(&trigger, NULL, task_work_race_worker, &trigger_arg); + if (err) { + ASSERT_OK(err, "pthread_create trigger"); + goto out; + } + trigger_started = true; + + for (i = 0; i < TASK_WORK_START_POLL_ITERS; i++) { + if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) + break; + usleep(TASK_WORK_POLL_US); + } + if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), + "thread start")) + goto out; + if (!ASSERT_OK(__atomic_load_n(&ctx.setup_err, __ATOMIC_ACQUIRE), + "thread affinity")) + goto out; + + skel->bss->trigger_tid = __atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE); + skel->bss->target_tid = __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE); + + /* Arm last, once this attempt's value, slots and TIDs are in place. */ + if (!ASSERT_OK(task_work_set_status(skel->maps.status, BUSY_ARM_SEQ, seq), + "arm")) + goto out; + + /* A fresh value has no expected BUSY source: check the first result. */ + err = task_work_wait_seq(skel->maps.status, BUSY_GEN1_RESULT, seq, &res); + if (!ASSERT_OK(err, "gen1 result")) + goto out; + if (!ASSERT_OK(res, "gen1 schedule")) { + fprintf(stderr, "attempt %d: gen1 failed: %d\n", seq, res); + goto out; + } + + if (!ASSERT_OK(task_work_busy_wait_window(skel, seq, &window), + "window close")) + goto out; + + if ((__u32)window == 3) { + /* + * The attempt did not arrive inside the wait bound. The + * elapsed time is a diagnostic only; count + * the miss and retry with a fresh attempt. + */ + result = TWB_MISSED; + goto out; + } + + /* + * Valid window: the handler saw this sequence's attempt before + * returning, so the attempt ran while the scheduling irq_work was + * still BUSY. The elapsed time is a diagnostic only: host vCPU + * preemption can stretch wall-clock time without affecting the + * gate behavior under test. + */ + err = task_work_wait_seq(skel->maps.status, BUSY_ACK, seq, &res); + if (!ASSERT_OK(err, "in-window attempt")) + goto out; + if (res != -EBUSY) { + fprintf(stderr, + "attempt %d: in-window reuse %s (%d), expected -EBUSY\n", + seq, res == 0 ? "accepted" : "failed", res); + ASSERT_EQ(res, -EBUSY, "in-window BUSY gate rejection"); + goto out; + } + + /* + * The window was released; retry until BUSY clears. The second + * callback must then run exactly once. + */ + if (!ASSERT_OK(task_work_set_status(skel->maps.status, BUSY_RECOVER_SEQ, seq), + "enable recovery")) + goto out; + err = task_work_wait_seq(skel->maps.status, BUSY_GEN2_RESULT, seq, &res); + if (!ASSERT_OK(err, "recovery result")) + goto out; + if (!ASSERT_OK(res, "recovery schedule")) { + fprintf(stderr, "attempt %d: recovery failed: %d\n", seq, res); + goto out; + } + if (!ASSERT_OK(task_work_busy_wait_value(skel, BUSY_CB2_SEQ, seq), + "gen2 callback")) + goto out; + result = TWB_VALID; +out: + __atomic_store_n(&ctx.stop, 1, __ATOMIC_RELEASE); + if (trigger_started && + (!ASSERT_OK(pthread_join(trigger, NULL), "trigger join") || + !ASSERT_OK(task_work_busy_wait_exit(ctx.trigger_tid), "trigger exit"))) { + cleanup_ok = false; + result = TWB_HARD_FAIL; + } + if (target_started && + (!ASSERT_OK(pthread_join(target, NULL), "target join") || + !ASSERT_OK(task_work_busy_wait_exit(ctx.target_tid), "target exit"))) { + cleanup_ok = false; + result = TWB_HARD_FAIL; + } + /* Both workers have passed exit_task_work; the count is now stable. */ + if (result == TWB_VALID) { + if (!ASSERT_OK(task_work_status(skel->maps.status, BUSY_CB2_COUNT, &count), + "cb2 count")) { + result = TWB_HARD_FAIL; + } else if (!ASSERT_EQ(count, 1, "gen2 callback exactly once")) { + fprintf(stderr, "attempt %d: gen2 callback count %lld\n", + seq, (long long)count); + result = TWB_HARD_FAIL; + } + } + /* + * Only a captured final window belongs to this attempt. The fexit + * runs in the trigger's interrupted context, so successful worker + * cleanup also completes its diagnostic write. Read once here; + * the elapsed value does not decide whether the attempt passed. + */ + if (cleanup_ok && window) { + err = task_work_status(skel->maps.status, BUSY_LAST_WAIT_NS, &wait_ns); + if (err) { + err = -errno; + if (result != TWB_HARD_FAIL) { + ASSERT_OK(err, "wait diagnostic"); + result = TWB_HARD_FAIL; + } else { + fprintf(stderr, + "attempt %d: elapsed diagnostic unavailable: %d\n", + seq, err); + } + } + } + *out_wait_ns = wait_ns; + fprintf(stderr, "attempt %d: %s (window %u wait_ns %lld)\n", + seq, twb_attempt_name(result), (__u32)window, (long long)wait_ns); + return result; +} + +void serial_test_task_work_busy(void) +{ + struct task_work_busy *skel; + struct bpf_link *link_tp = NULL, *link_fexit = NULL; + cpu_set_t allowed; + int cpu, cpu_count = 0, trigger_cpu = -1, target_cpu = -1; + int attempt, attempts_started = 0, valid = 0, missed = 0; + bool hard_fail = false; + __s64 max_wait_ns = 0; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) { + ASSERT_OK(-errno, "sched_getaffinity"); + return; + } + for (cpu = 0; cpu < CPU_SETSIZE && cpu_count < 2; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + if (!cpu_count) + trigger_cpu = cpu; + else + target_cpu = cpu; + cpu_count++; + } + if (cpu_count < 2) { + printf("%s:SKIP:need two CPUs in the process affinity mask\n", + __func__); + test__skip(); + return; + } + + skel = task_work_busy__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + if (skel->kconfig->CONFIG_PREEMPT_RT) { + printf("%s:SKIP:CONFIG_PREEMPT_RT=y runs irq_work in a kernel thread\n", + __func__); + test__skip(); + goto cleanup; + } + + link_tp = bpf_program__attach_tracepoint(skel->progs.busy_sched_work, + "syscalls", + "sys_enter_getppid"); + if (!ASSERT_OK_PTR(link_tp, "attach tracepoint")) + goto cleanup; + + link_fexit = bpf_program__attach_trace(skel->progs.busy_fexit_task_work_add); + if (!ASSERT_OK_PTR(link_fexit, "attach fexit")) + goto cleanup; + + for (attempt = 1; + attempt <= TASK_WORK_BUSY_MAX_ATTEMPTS && + valid < TASK_WORK_BUSY_ROUNDS; + attempt++) { + enum task_work_busy_attempt res; + __s64 wait_ns = 0; + + attempts_started = attempt; + res = task_work_busy_run_attempt(skel, attempt, trigger_cpu, + target_cpu, &wait_ns); + switch (res) { + case TWB_VALID: + valid++; + if (wait_ns > max_wait_ns) + max_wait_ns = wait_ns; + break; + case TWB_MISSED: + missed++; + break; + case TWB_HARD_FAIL: + hard_fail = true; + break; + } + if (hard_fail) + break; + } + + fprintf(stderr, + "task_work_busy: %d/%d valid, %d missed, %d attempts, max_wait %lld ns\n", + valid, TASK_WORK_BUSY_ROUNDS, missed, attempts_started, + (long long)max_wait_ns); + + /* + * The coverage assertion only makes sense when no hard + * failure aborted the loop; after a hard failure keep the primary + * reason instead of deriving more assertion noise from it. + */ + if (!hard_fail) + ASSERT_EQ(valid, TASK_WORK_BUSY_ROUNDS, "valid busy windows"); +cleanup: + bpf_link__destroy(link_tp); + bpf_link__destroy(link_fexit); + task_work_busy__destroy(skel); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c b/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c index 462512f..1391978 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c
@@ -90,7 +90,7 @@ static void cleanup(void) remove_netns(SERVER_NS); } -static void run_test(void) +static void run_test(struct test_tc_edt *skel) { int server_fd, client_fd, err; double rate_mbps, rate_error; @@ -113,12 +113,19 @@ static void run_test(void) if (!ASSERT_OK_FD(client_fd, "connect client")) return; - ts_start = get_time_ns(); err = send_recv_data(server_fd, client_fd, TX_BYTES_COUNT); ts_end = get_time_ns(); close_netns(nstoken); ASSERT_OK(err, "send_recv_data"); + /* + * Start from the first packet with data rather than before + * send_recv_data(), which also sets up the sender. + */ + ts_start = skel->bss->tx_start_ns; + if (!ASSERT_GT(ts_start, 0, "tx_start_ns")) + return; + rate_mbps = TX_BYTES_COUNT / ((ts_end - ts_start) / 1000.0); rate_error = fabs((rate_mbps - TARGET_RATE_MBPS) * 100.0 / TARGET_RATE_MBPS); @@ -138,7 +145,7 @@ void test_tc_edt(void) if (!ASSERT_OK(setup(skel), "global setup")) return; - run_test(); + run_test(skel); cleanup(); test_tc_edt__destroy(skel);
diff --git a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c index eb93099..7b35bf9 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c
@@ -161,9 +161,9 @@ static int add_vxlan_tunnel(void) IP4_ADDR2_VETH1, MAC_VETH1); /* root namespace */ - SYS(fail, "ip link add dev %s type vxlan external gbp dstport 4789", - VXLAN_TUNL_DEV1); - SYS(fail, "ip link set dev %s address %s up", VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip link add dev %s address %s type vxlan external gbp dstport 4789", + VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip link set dev %s up", VXLAN_TUNL_DEV1); SYS(fail, "ip addr add dev %s %s/24", VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); SYS(fail, "ip neigh add %s lladdr %s dev %s", IP4_ADDR_TUNL_DEV0, MAC_TUNL_DEV0, VXLAN_TUNL_DEV1); @@ -198,11 +198,10 @@ static int add_ip6vxlan_tunnel(void) IP6VXLAN_TUNL_DEV0, MAC_TUNL_DEV0); /* root namespace */ - SYS(fail, "ip link add dev %s type vxlan external dstport 4789", - IP6VXLAN_TUNL_DEV1); - SYS(fail, "ip addr add dev %s %s/24", IP6VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); - SYS(fail, "ip link set dev %s address %s up", + SYS(fail, "ip link add dev %s address %s type vxlan external dstport 4789", IP6VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip addr add dev %s %s/24", IP6VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); + SYS(fail, "ip link set dev %s up", IP6VXLAN_TUNL_DEV1); return 0; fail:
diff --git a/tools/testing/selftests/bpf/prog_tests/timer.c b/tools/testing/selftests/bpf/prog_tests/timer.c index 09ff21e1..1782231 100644 --- a/tools/testing/selftests/bpf/prog_tests/timer.c +++ b/tools/testing/selftests/bpf/prog_tests/timer.c
@@ -262,6 +262,34 @@ static int timer_cancel_async(struct timer *timer_skel) return 0; } +/* + * A timer callback which re-arms itself and reaches a loop through a call: + * the two entries into the callback meet at the loop, in a frame of its own. + */ +static int timer_loop_rearm(struct timer *timer_skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd, i; + + err = timer__attach(timer_skel); + if (!ASSERT_OK(err, "timer_attach")) + return err; + + timer_skel->bss->loop_rearm = 1; + + prog_fd = bpf_program__fd(timer_skel->progs.test_loop_rearm); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run")) + return err; + + for (i = 0; i < 100 && timer_skel->bss->loop_sum < 120 * 2; i++) + usleep(1000); + + timer__detach(timer_skel); + ASSERT_EQ(timer_skel->bss->loop_sum, 120 * 2, "loop_sum"); + return 0; +} + static void test_timer(int (*timer_test_fn)(struct timer *timer_skel)) { struct timer *timer_skel = NULL; @@ -287,6 +315,11 @@ void serial_test_timer(void) RUN_TESTS(timer_failure); } +void serial_test_timer_loop_rearm(void) +{ + test_timer(timer_loop_rearm); +} + void serial_test_timer_stress(void) { test_timer(timer_stress);
diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index 0aa9532..cc72701 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c
@@ -662,6 +662,97 @@ static void test_attach_api_fails(void) free(ids2); } +/* + * Find two BTF ids that resolve to the same address. The stub of a syscall + * without arguments is an alias of its __do_sys_ function. + */ +static bool get_same_addr_ids(__u32 *ids) +{ + const char *prefix = "__x64_sys_"; + __u32 nr, type_id; + bool found = false; + struct btf *btf; + char alias[128]; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return false; + + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr && !found; type_id++) { + const struct btf_type *type = btf__type_by_id(btf, type_id); + const char *str; + long addr; + __s32 id; + + if (!btf_is_func(type)) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str || strncmp(str, prefix, strlen(prefix))) + continue; + + snprintf(alias, sizeof(alias), "__do_sys_%s", str + strlen(prefix)); + id = btf__find_by_name_kind(btf, alias, BTF_KIND_FUNC); + if (id <= 0) + continue; + + addr = ksym_get_addr(str); + if (!addr || addr != ksym_get_addr(alias)) + continue; + + ids[0] = type_id; + ids[1] = id; + found = true; + } + + btf__free(btf); + return found; +} + +static void test_attach_same_addr(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_fail *skel; + struct bpf_link *link; + __u32 ids[2]; + int i; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + if (!get_same_addr_ids(ids)) { + test__skip(); + return; + } + + skel = tracing_multi_fail__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_fail__open_and_load")) + return; + + /* Each id attaches fine on its own.. */ + for (i = 0; i < 2; i++) { + opts.ids = &ids[i]; + opts.cnt = 1; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_OK_PTR(link, "attach_single")) + goto cleanup; + bpf_link__destroy(link); + } + + /* ..but there's only one call site, so not both of them. */ + opts.ids = ids; + opts.cnt = 2; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "attach_same_addr")) + bpf_link__destroy(link); + +cleanup: + tracing_multi_fail__destroy(skel); +} + void serial_test_tracing_multi_bench_attach(void) { LIBBPF_OPTS(bpf_tracing_multi_opts, opts); @@ -748,6 +839,12 @@ void serial_test_tracing_multi_bench_attach(void) if (!btf_type_is_traceable_func(btf, type)) continue; + /* + * BTF can have more functions of the same name and they all + * resolve to the same address, take just the first one. + */ + tdelete(&str, &root, compare); + err = libbpf_ensure_mem((void **) &ids, &cap, sizeof(*ids), cnt + 1); if (err) goto cleanup; @@ -1023,6 +1120,8 @@ void test_tracing_multi_test(void) test_session(); if (test__start_subtest("attach_api_fails")) test_attach_api_fails(); + if (test__start_subtest("attach_same_addr")) + test_attach_same_addr(); RUN_TESTS(tracing_multi_verifier); if (test__start_subtest("fentry_after_multi")) test_fentry_after_multi();
diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 15b95d0..de358f3 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c
@@ -5,6 +5,10 @@ #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" #include "tracing_struct_int128.skel.h" +#include "tracing_struct_int128_tgt.skel.h" +#include "tracing_struct_int128_tracer.skel.h" +#include "tracing_stack_arg_tailcall_tgt.skel.h" +#include "tracing_stack_arg_tailcall_tracer.skel.h" static void test_struct_args(void) { @@ -117,9 +121,11 @@ static void test_int128_args(void) { /* * __int128 arguments are passed in a register pair on x86_64 and - * arm64, which the trampoline packs into two context slots. Other - * architectures pass a __int128 differently (e.g. s390x passes larger - * arguments by reference), so only exercise this on x86_64 and arm64. + * arm64, which the trampoline packs into two context slots. Past the + * registers, both start an argument aligned to 16 bytes at an even + * stack slot, which the trampoline packs too. Other architectures + * pass a __int128 differently (e.g. s390x passes larger arguments by + * reference), so only exercise this on x86_64 and arm64. */ #if defined(__x86_64__) || defined(__aarch64__) struct tracing_struct_int128 *skel; @@ -139,6 +145,20 @@ static void test_int128_args(void) ASSERT_EQ(skel->bss->t_c, 3, "t:c"); ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + ASSERT_EQ(skel->bss->s_g, 7, "s:g"); + ASSERT_EQ(skel->bss->s_h_lo, 8, "s:h.lo"); + ASSERT_EQ(skel->bss->s_h_hi, 9, "s:h.hi"); + ASSERT_EQ(skel->bss->s_i, 10, "s:i"); + ASSERT_EQ(skel->bss->s_ret, 385, "s ret"); + + ASSERT_EQ(skel->bss->b_f_a, 6, "b:f.a"); + ASSERT_EQ(skel->bss->b_f_b, 7, "b:f.b"); + ASSERT_EQ(skel->bss->b_g, 8, "b:g"); + ASSERT_EQ(skel->bss->b_h, 9, "b:h"); + ASSERT_EQ(skel->bss->b_i_lo, 10, "b:i.lo"); + ASSERT_EQ(skel->bss->b_i_hi, 11, "b:i.hi"); + ASSERT_EQ(skel->bss->b_ret, 506, "b ret"); + destroy_skel: tracing_struct_int128__destroy(skel); #else @@ -146,6 +166,191 @@ static void test_int128_args(void) #endif } +static void test_int128_tgt_prog(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) || defined(__s390x__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_struct_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.odd_entry, fd, "callee_odd"); + err = err ?: bpf_program__set_attach_target(tracer->progs.odd_exit, fd, "callee_odd"); + if (tgt->rodata->has_stack_arg) { + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_entry, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_entry, fd, + "callee_split"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_exit, fd, + "callee_split"); + } else { + bpf_program__set_autoload(tracer->progs.stack_entry, false); + bpf_program__set_autoload(tracer->progs.stack_exit, false); + bpf_program__set_autoload(tracer->progs.split_entry, false); + bpf_program__set_autoload(tracer->progs.split_exit, false); + } + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_struct_int128_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->odd_a, 1, "odd:a"); + ASSERT_EQ(tracer->bss->odd_lo, 8, "odd:v.lo"); + ASSERT_EQ(tracer->bss->odd_hi, 9, "odd:v.hi"); + ASSERT_EQ(tracer->bss->odd_b, 2, "odd:b"); + ASSERT_EQ(tgt->bss->ret_odd, 52, "odd ret"); + ASSERT_EQ(tracer->bss->odd_ret, 52, "odd fexit ret"); + + if (tgt->rodata->has_stack_arg) { + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_lo, 8, "stack:h.lo"); + ASSERT_EQ(tracer->bss->stack_hi, 9, "stack:h.hi"); + ASSERT_EQ(tracer->bss->stack_i, 10, "stack:i"); + ASSERT_EQ(tgt->bss->ret_stack, 385, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 385, "stack fexit ret"); + + ASSERT_EQ(tracer->bss->split_e, 5, "split:e"); + ASSERT_EQ(tracer->bss->split_lo, 8, "split:h.lo"); + ASSERT_EQ(tracer->bss->split_hi, 9, "split:h.hi"); + ASSERT_EQ(tracer->bss->split_i, 10, "split:i"); + ASSERT_EQ(tgt->bss->ret_split, 246, "split ret"); + ASSERT_EQ(tracer->bss->split_ret, 246, "split fexit ret"); + } + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + +/* + * A static function whose BTF is unreliable gets a trampoline that only knows + * of the register arguments, so one taking stack arguments must be refused. + */ +static void test_tgt_prog_unreliable(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + char log_buf[4096] = {}; + struct bpf_program *prog; + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + bpf_object__for_each_program(prog, tracer->obj) + bpf_program__set_autoload(prog, false); + bpf_program__set_autoload(tracer->progs.unreliable_entry, true); + bpf_program__set_log_buf(tracer->progs.unreliable_entry, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(tracer->progs.unreliable_entry, fd, + "callee_unreliable"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + ASSERT_ERR(err, "tracer__load"); + ASSERT_HAS_SUBSTR(log_buf, "unreliable BTF and stack arguments", "log"); + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + +/* + * A trampoline on a function of a tail_call_reachable program keeps the tail + * call counter pointer on its stack, and has to hand the function its stack + * arguments past it. + */ +static void test_stack_arg_tailcall(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_stack_arg_tailcall_tracer *tracer = NULL; + struct tracing_stack_arg_tailcall_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_stack_arg_tailcall_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_stack_arg_tailcall_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.stack_entry, fd, "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, "callee_stack"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_stack_arg_tailcall_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_stack_arg_tailcall_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_h, 8, "stack:h"); + ASSERT_EQ(tgt->bss->ret_stack, 204, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 204, "stack fexit ret"); + +out: + tracing_stack_arg_tailcall_tracer__destroy(tracer); + tracing_stack_arg_tailcall_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -181,6 +386,12 @@ void test_tracing_struct(void) test_struct_many_args(); if (test__start_subtest("int128_args")) test_int128_args(); + if (test__start_subtest("int128_tgt_prog")) + test_int128_tgt_prog(); + if (test__start_subtest("tgt_prog_unreliable")) + test_tgt_prog_unreliable(); + if (test__start_subtest("stack_arg_tailcall")) + test_stack_arg_tailcall(); if (test__start_subtest("union_args")) test_union_args(); }
diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_attach.c b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c new file mode 100644 index 0000000..37e03b3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c
@@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include "verifier_unwind.skel.h" +#include "unwind_attach.skel.h" + +static char log_buf[16 * 1024]; + +static int run(int fd) +{ + char data[64] = {}; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = data, + .data_size_in = sizeof(data), + .repeat = 1, + ); + + ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "test_run"); + return opts.retval; +} + +/* + * msg: why it cannot be loaded. Otherwise retval is what the target returns + * with the prog attached. + */ +static void attach(int tgt_fd, const char *func, const char *name, const char *msg, int retval) +{ + struct bpf_link *link = NULL; + struct unwind_attach *skel; + struct bpf_program *prog; + int err; + + skel = unwind_attach__open(); + if (!ASSERT_OK_PTR(skel, "unwind_attach__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, name); + if (!ASSERT_OK_PTR(prog, name)) + goto out; + bpf_program__set_autoload(prog, true); + log_buf[0] = 0; + bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(prog, tgt_fd, func); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = unwind_attach__load(skel); + if (msg) { + ASSERT_ERR(err, name); + ASSERT_HAS_SUBSTR(log_buf, msg, name); + goto out; + } + if (!ASSERT_OK(err, name)) + goto out; + + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "attach")) + goto out; + ASSERT_EQ(run(tgt_fd), retval, "retval"); + if (prog == skel->progs.fentry_unwind) + ASSERT_EQ(skel->bss->fentry_cnt, 1, "fentry_cnt"); +out: + bpf_link__destroy(link); + unwind_attach__destroy(skel); +} + +void test_unwind_attach(void) +{ + struct verifier_unwind *tgt; + struct bpf_program *prog; + int tgt_fd, plain_fd, shared_fd, tail_fd, throws_fd, err, key = 0; + + if (!is_jit_enabled()) { + printf("%s:SKIP:JIT is disabled\n", __func__); + test__skip(); + return; + } + tgt = verifier_unwind__open(); + if (!ASSERT_OK_PTR(tgt, "verifier_unwind__open")) + return; + bpf_object__for_each_program(prog, tgt->obj) + bpf_program__set_autoload(prog, prog == tgt->progs.calls_global_unwinds || + prog == tgt->progs.calls_global_never_unwinds || + prog == tgt->progs.shared_pad || + prog == tgt->progs.tail_call_in_subprog || + prog == tgt->progs.exception_callback_unwinds); + err = verifier_unwind__load(tgt); + if (err == -ENOTSUPP) { + printf("%s:SKIP:JIT does not support unwinding\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "verifier_unwind__load")) + goto out; + /* + * It calls global_unwinds(), which unwinds with r0 = 5, and returns r0 + 10 + * from the pad. + */ + tgt_fd = bpf_program__fd(tgt->progs.calls_global_unwinds); + plain_fd = bpf_program__fd(tgt->progs.calls_global_never_unwinds); + ASSERT_EQ(run(tgt_fd), 15, "retval"); + + attach(tgt_fd, "global_unwinds", "fentry_unwind", NULL, 15); + /* unwind_exit would return to the trampoline */ + attach(tgt_fd, "global_unwinds", "fexit_unwind", + "Only fentry can attach to global_unwinds", 0); + /* An extension leaves either way, like what it replaced */ + attach(tgt_fd, "global_unwinds", "freplace_returns", NULL, 9); + attach(tgt_fd, "global_unwinds", "freplace_unwinds", NULL, 13); + /* There is no unwind_goto after the call */ + attach(plain_fd, "global_never_unwinds", "freplace_unwinds", + "Extension program can unwind, while original does not", 0); + ASSERT_EQ(run(tgt_fd), 15, "retval"); + + /* + * An exception callback returns from the frame of the main prog. Here + * that's a return to a call that unwind_goto follows. unwind_exit of + * the callback has to be exit: the pad wasn't verified with it. + */ + shared_fd = bpf_program__fd(tgt->progs.shared_pad); + attach(shared_fd, "global_never_unwinds", "freplace_throws", NULL, 3); + + tail_fd = bpf_program__fd(tgt->progs.tail_call_in_subprog); + throws_fd = bpf_program__fd(tgt->progs.exception_callback_unwinds); + err = bpf_map__update_elem(tgt->maps.prog_array, &key, sizeof(key), + &throws_fd, sizeof(throws_fd), 0); + if (ASSERT_OK(err, "prog_array")) + ASSERT_EQ(run(tail_fd), 7, "retval"); +out: + verifier_unwind__destroy(tgt); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c b/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c new file mode 100644 index 0000000..9faba99 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c
@@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +/* The progs are under #ifdef __BPF_FEATURE_UNWIND */ +static bool llvm_lowers_unwinding(const char *file) +{ + struct btf *btf = btf__parse(file, NULL); + bool ret; + + ret = btf && btf__find_by_name_kind(btf, "catches", BTF_KIND_FUNC) > 0; + btf__free(btf); + return ret; +} + +static void run(struct bpf_object *obj, const char *name, int len, int retval, int dtors) +{ + struct bpf_program *prog = bpf_object__find_program_by_name(obj, name); + struct bpf_map *map = bpf_object__find_map_by_name(obj, ".data.dtors"); + char data[256] = {}; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = data, + .data_size_in = len, + .repeat = 1, + ); + int key = 0, val = 0; + + if (!ASSERT_OK_PTR(prog, name) || !ASSERT_OK_PTR(map, "dtors")) + return; + ASSERT_OK(bpf_map__update_elem(map, &key, sizeof(key), &val, sizeof(val), 0), "clear"); + ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run"); + ASSERT_EQ(opts.retval, retval, "retval"); + ASSERT_OK(bpf_map__lookup_elem(map, &key, sizeof(key), &val, sizeof(val), 0), "lookup"); + ASSERT_EQ(val, dtors, "dtors"); +} + +void test_unwind_cxx(void) +{ + const char *file = "./unwind_cxx.bpf.o"; + struct bpf_object *obj; + struct bpf_program *prog; + int err = -EINVAL; + + if (access(file, R_OK)) { + printf("%s:SKIP:gcc doesn't build %s\n", __func__, file); + test__skip(); + return; + } + if (!is_jit_enabled()) { + printf("%s:SKIP:JIT is disabled\n", __func__); + test__skip(); + return; + } + if (!llvm_lowers_unwinding(file)) { + printf("%s:SKIP:llvm doesn't lower unwinding\n", __func__); + test__skip(); + return; + } + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open")) + return; + prog = bpf_object__find_program_by_name(obj, "catches_aggregate"); + if (!ASSERT_OK_PTR(prog, "catches_aggregate")) + goto out; + bpf_program__set_autoload(prog, false); + err = bpf_object__load(obj); + if (err == -ENOTSUPP) { + printf("%s:SKIP:JIT does not support unwinding\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "load")) + goto out; + + /* inner's destructor, then middle's */ + run(obj, "catches", 64, 65, 12); + run(obj, "catches", 200, 42, 12); + run(obj, "does_not_catch", 64, 65, 12); + run(obj, "does_not_catch", 200, 0, 12); +out: + bpf_object__close(obj); + if (err) + return; + + if (!test__start_subtest("aggregate_return")) + return; + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open_aggregate")) + return; + bpf_object__for_each_program(prog, obj) + bpf_program__set_autoload(prog, + strcmp(bpf_program__name(prog), "catches_aggregate") == 0); + if (!ASSERT_OK(bpf_object__load(obj), "load_aggregate")) + goto out_aggregate; + run(obj, "catches_aggregate", 64, 71, 0); + run(obj, "catches_aggregate", 200, 42, 0); +out_aggregate: + bpf_object__close(obj); +}
diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c index f0baf57..7fbcfdd 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
@@ -94,14 +94,14 @@ static int child_func(void *arg) /* wait for parent's kick */ err = read(child->go[0], &c, 1); if (err != 1) - exit(err); + _exit(err); uprobe_multi_func_1(); uprobe_multi_func_2(); uprobe_multi_func_3(); usdt_trigger(); - exit(errno); + _exit(errno); } static int spawn_child_flag(struct child *child, bool clone_vm)
diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 8004c95..eff1e57 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c
@@ -250,6 +250,7 @@ static void subtest_basic_usdt(bool optimized) #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_2_cross_page(void); extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; @@ -343,6 +344,57 @@ static void subtest_optimized_attach(void) } /* + * Test attachment to a USDT probe whose nop10 crosses a page boundary. + * The kernel can't optimize such nop10, so libbpf keeps the uprobe on + * the preceding 1-byte nop. Do not assume any particular placement + * here, though: however the probe ends up attached, the attachment + * must succeed and the probe must fire. + */ +static void subtest_optimized_attach_cross_page(void) +{ + long page_sz = sysconf(_SC_PAGESIZE); + struct test_usdt *skel; + __u8 *addr = NULL; + long i; + + /* combo is placed up to a page of padding after the function start */ + for (i = 0; i < 2 * page_sz; i++) { + if (!memcmp((void *)usdt_2_cross_page + i, nop1_nop10_combo, 11)) { + addr = (void *)usdt_2_cross_page + i; + break; + } + } + if (!ASSERT_OK_PTR(addr, "find_nop1_nop10_combo")) + return; + + /* layout sanity check: the nop10 must cross the page boundary */ + if (!ASSERT_GT((unsigned long)(addr + 1) % page_sz + 10, page_sz, + "nop10_crosses_page")) + return; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load")) + return; + + skel->bss->my_pid = getpid(); + + skel->links.usdt0 = bpf_program__attach_usdt(skel->progs.usdt0, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", + "usdt_2_cross_page", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt0, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_2_cross_page(); + usdt_2_cross_page(); + + ASSERT_EQ(skel->bss->usdt0_called, 2, "usdt0_called"); + +cleanup: + test_usdt__destroy(skel); +} + +/* * Test that USDT arguments survive nop10 optimization in a function where * the compiler places operands in the red zone. * @@ -660,6 +712,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_attach_cross_page")) + subtest_optimized_attach_cross_page(); if (test__start_subtest("optimized_red_zone")) subtest_optimized_red_zone(); #endif
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8b439e1..04740ba 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -3,12 +3,17 @@ #include <test_progs.h> #include "arena_kfunc.skel.h" +#include "arena_noarena.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "cb_refs.skel.h" +#include "verifier_aggregate_arg.skel.h" +#include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" +#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -16,16 +21,20 @@ #include "verifier_basic_stack.skel.h" #include "verifier_bitfield_write.skel.h" #include "verifier_bounds.skel.h" +#include "verifier_bounds_step.skel.h" #include "verifier_bounds_deduction.skel.h" #include "verifier_bounds_deduction_non_const.skel.h" #include "verifier_bounds_mix_sign_unsign.skel.h" #include "verifier_bpf_get_stack.skel.h" #include "verifier_bpf_trap.skel.h" #include "verifier_bswap.skel.h" +#include "verifier_btf_array_access.skel.h" #include "verifier_btf_ctx_access.skel.h" #include "verifier_btf_flex_array.skel.h" #include "verifier_btf_unreliable_prog.skel.h" #include "verifier_call_large_imm.skel.h" +#include "verifier_callx.skel.h" +#include "verifier_callx_rodata.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -50,18 +59,24 @@ #include "verifier_helper_packet_access.skel.h" #include "verifier_helper_restricted.skel.h" #include "verifier_helper_value_access.skel.h" +#include "verifier_idoms.skel.h" #include "verifier_int_ptr.skel.h" #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" +#include "verifier_kfunc_packet_access.skel.h" +#include "verifier_kfunc_uninit.skel.h" +#include "verifier_kfunc_uninit_multi.skel.h" #include "verifier_kfunc_perfmon.skel.h" #include "verifier_ld_ind.skel.h" +#include "verifier_large_stack.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" #include "verifier_linked_scalars.skel.h" #include "verifier_live_stack.skel.h" #include "verifier_liveness_exp.skel.h" #include "verifier_load_acquire.skel.h" +#include "verifier_loop_hierarchy.skel.h" #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" @@ -94,6 +109,7 @@ #include "verifier_ringbuf.skel.h" #include "verifier_runtime_jit.skel.h" #include "verifier_scalar_ids.skel.h" +#include "verifier_scev.skel.h" #include "verifier_sdiv.skel.h" #include "verifier_search_pruning.skel.h" #include "verifier_sock.skel.h" @@ -114,6 +130,7 @@ #include "verifier_typedef.skel.h" #include "verifier_uninit.skel.h" #include "verifier_unpriv.skel.h" +#include "verifier_unwind.skel.h" #include "verifier_unpriv_perf.skel.h" #include "verifier_value_adj_spill.skel.h" #include "verifier_value.skel.h" @@ -129,6 +146,7 @@ #include "verifier_bits_iter.skel.h" #include "verifier_set_retval.skel.h" #include "verifier_lsm.skel.h" +#include "verifier_lsm_init_xattr.skel.h" #include "verifier_jit_inline.skel.h" #include "irq.skel.h" #include "verifier_ctx_ptr_param.skel.h" @@ -168,29 +186,47 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +/* for tests of what the interpreter doesn't support */ +#define RUN_JITED(skel) do { \ + if (is_jit_enabled()) \ + RUN(skel); \ + else \ + test__skip(); \ +} while (0) + void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } +void test_arena_noarena(void) { RUN_TESTS(arena_noarena); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_cb_refs(void) { RUN_TESTS(cb_refs); } + +void test_verifier_aggregate_arg(void) { RUN_TESTS(verifier_aggregate_arg); } +void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } +void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } void test_verifier_bitfield_write(void) { RUN(verifier_bitfield_write); } void test_verifier_bounds(void) { RUN(verifier_bounds); } +void test_verifier_bounds_step(void) { RUN(verifier_bounds_step); } void test_verifier_bounds_deduction(void) { RUN(verifier_bounds_deduction); } void test_verifier_bounds_deduction_non_const(void) { RUN(verifier_bounds_deduction_non_const); } void test_verifier_bounds_mix_sign_unsign(void) { RUN(verifier_bounds_mix_sign_unsign); } void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); } void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } void test_verifier_bswap(void) { RUN(verifier_bswap); } +void test_verifier_btf_array_access(void) { RUN(verifier_btf_array_access); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } void test_verifier_btf_flex_array(void) { RUN(verifier_btf_flex_array); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } +void test_verifier_callx(void) { RUN_JITED(verifier_callx); } +void test_verifier_callx_rodata(void) { RUN_JITED(verifier_callx_rodata); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } @@ -215,18 +251,24 @@ void test_verifier_helper_access_var_len(void) { RUN(verifier_helper_access_var_ void test_verifier_helper_packet_access(void) { RUN(verifier_helper_packet_access); } void test_verifier_helper_restricted(void) { RUN(verifier_helper_restricted); } void test_verifier_helper_value_access(void) { RUN(verifier_helper_value_access); } +void test_verifier_idoms(void) { RUN(verifier_idoms); } void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } +void test_verifier_kfunc_packet_access(void) { RUN_TESTS(verifier_kfunc_packet_access); } +void test_verifier_kfunc_uninit(void) { RUN_TESTS(verifier_kfunc_uninit); } +void test_verifier_kfunc_uninit_multi(void) { RUN_TESTS(verifier_kfunc_uninit_multi); } void test_verifier_kfunc_perfmon(void) { RUN(verifier_kfunc_perfmon); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } +void test_verifier_large_stack(void) { RUN(verifier_large_stack); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); } void test_verifier_linked_scalars(void) { RUN(verifier_linked_scalars); } void test_verifier_live_stack(void) { RUN(verifier_live_stack); } void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } +void test_verifier_loop_hierarchy(void) { RUN(verifier_loop_hierarchy); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } @@ -258,6 +300,7 @@ void test_verifier_regalloc(void) { RUN(verifier_regalloc); } void test_verifier_ringbuf(void) { RUN(verifier_ringbuf); } void test_verifier_runtime_jit(void) { RUN(verifier_runtime_jit); } void test_verifier_scalar_ids(void) { RUN(verifier_scalar_ids); } +void test_verifier_scev(void) { RUN(verifier_scev); } void test_verifier_sdiv(void) { RUN(verifier_sdiv); } void test_verifier_search_pruning(void) { RUN(verifier_search_pruning); } void test_verifier_sock(void) { RUN(verifier_sock); } @@ -278,6 +321,7 @@ void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); } void test_verifier_typedef(void) { RUN(verifier_typedef); } void test_verifier_uninit(void) { RUN(verifier_uninit); } void test_verifier_unpriv(void) { RUN(verifier_unpriv); } +void test_verifier_unwind(void) { RUN(verifier_unwind); } void test_verifier_unpriv_perf(void) { RUN(verifier_unpriv_perf); } void test_verifier_value_adj_spill(void) { RUN(verifier_value_adj_spill); } void test_verifier_value(void) { RUN(verifier_value); } @@ -292,6 +336,7 @@ void test_verifier_xdp_direct_packet_access(void) { RUN(verifier_xdp_direct_pack void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); } void test_verifier_set_retval(void) { RUN(verifier_set_retval); } void test_verifier_lsm(void) { RUN(verifier_lsm); } +void test_verifier_lsm_init_xattr(void) { RUN(verifier_lsm_init_xattr); } void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); }
diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index f327feb..12b146d 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c
@@ -257,7 +257,7 @@ static void test_verify_pkcs7_sig_from_map(void) if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp")) return; - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto close_prog; @@ -458,7 +458,7 @@ static void test_pkcs7_sig_fsverity(void) snprintf(data_path, PATH_MAX, "%s/data-file", tmp_dir); snprintf(sig_path, PATH_MAX, "%s/sig-file", tmp_dir); - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto out;
diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 84831ee..97926e3 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c
@@ -64,10 +64,9 @@ static void test_failure_map_no_btf(void) goto out; err = bpf_map__reuse_fd(skel->maps.array, map_fd); - if (!ASSERT_OK(err, "map reuse fd")) { - close(map_fd); + close(map_fd); + if (!ASSERT_OK(err, "map reuse fd")) goto out; - } insns = bpf_program__insns(skel->progs.test_map_no_btf); if (!ASSERT_OK_PTR(insns, "insns ptr"))
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c index fce2036..a894b1a 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c
@@ -18,6 +18,9 @@ static void test_xdp_update_frags(void) return; prog = bpf_object__next_program(obj, NULL); + ASSERT_EQ(bpf_program__flags(prog) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + if (bpf_object__load(obj)) return;
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c index a8ab052..dff6b3e 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c
@@ -146,6 +146,11 @@ static void test_xdp_with_devmap_frags_helpers(void) if (!ASSERT_OK_PTR(skel, "test_xdp_with_devmap_helpers__open_and_load")) return; + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm_frags) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm) & BPF_F_XDP_HAS_FRAGS, + 0, "frags not in program flags"); + dm_fd_frags = bpf_program__fd(skel->progs.xdp_dummy_dm_frags); map_fd = bpf_map__fd(skel->maps.dm_ports); err = bpf_prog_get_info_by_fd(dm_fd_frags, &info, &len);
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 5c31054..3ab9634 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c
@@ -408,14 +408,14 @@ void test_xdp_metadata(void) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, rx_ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); /* Make sure we can load a dev-bound program that performs * XDP_REDIRECT into a devmap. */ new_prog = bpf_object__find_program_by_name(bpf_obj->obj, "redirect"); bpf_program__set_ifindex(new_prog, rx_ifindex); - bpf_program__set_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton")) goto out;
diff --git a/tools/testing/selftests/bpf/prog_tests/xsk.c b/tools/testing/selftests/bpf/prog_tests/xsk.c index 6e2f63e..772f3dd 100644 --- a/tools/testing/selftests/bpf/prog_tests/xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/xsk.c
@@ -75,7 +75,7 @@ static void test_xsk(const struct test_spec *test_to_run, enum test_mode mode) if (!ASSERT_OK_PTR(ifobj_rx, "create ifobj_rx")) goto delete_tx; - if (!ASSERT_OK(configure_ifobj(ifobj_tx, ifobj_rx), "conigure ifobj")) + if (!ASSERT_OK(configure_ifobj(ifobj_tx, ifobj_rx), "configure ifobj")) goto delete_rx; ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_func.c b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c new file mode 100644 index 0000000..2132b92 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c
@@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +struct pair { + __u64 lo; + __u64 hi; +}; + +struct too_big { + __u64 a; + __u64 b; + __u64 c; +}; + +#if defined(__clang__) + +__noinline __u64 global_arg_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0x33) +__naked int aggregate_arg_pair_asm(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 0x10;" /* p.lo */ + "r3 = 0x20;" /* p.hi */ + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +SEC("tc") +__failure __msg("R2 type=fp expected=scalar") +__naked int aggregate_arg_pair_ptr_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = r10;" /* a stack pointer where p.lo belongs */ + "r3 = 0x20;" + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +#endif + +__noinline __u64 global_arg_too_big(struct too_big s) +{ + return s.a + s.b + s.c; +} + +SEC("tc") +__failure __msg("in global_arg_too_big() has size 24, only 1 to 16 bytes can be passed by value") +__naked int aggregate_arg_too_big_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "call %[global_arg_too_big];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_too_big) + : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +/* + * One slot per parameter, so the slot count is settled before the parameters + * are walked. The cases below reach the same count only once they have been. + */ +__noinline __u64 global_arg_six_scalars(int a, int b, int c, int d, int e, int f) +{ + return (__u64)a + b + c + d + e + f; +} + +SEC("tc") +__failure __msg("global function global_arg_six_scalars() needs 6 > 5 argument slots") +__naked int aggregate_arg_six_scalars_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_six_scalars];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_six_scalars) + : __clobber_all); +} + +__noinline __u64 global_arg_split(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_split() needs 6 > 5 argument slots") +__naked int aggregate_arg_split_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_split];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_split) + : __clobber_all); +} + +__noinline __u64 global_arg_past_regs(struct pair p, struct pair q, int a, struct pair r) +{ + return p.lo + p.hi + q.lo + q.hi + a + r.lo + r.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_past_regs() needs 7 > 5 argument slots") +__naked int aggregate_arg_past_regs_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_past_regs];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_past_regs) + : __clobber_all); +} + +#ifdef __SIZEOF_INT128__ + +__noinline __u64 global_arg_i128_slots(u128 v, int a, int b, int c, int d) +{ + return (__u64)v + a + b + c + d; +} + +SEC("tc") +__failure __msg("global function global_arg_i128_slots() needs 6 > 5 argument slots") +__naked int aggregate_arg_i128_slots_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_i128_slots];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_i128_slots) + : __clobber_all); +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c new file mode 100644 index 0000000..c87e886 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c
@@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +#if defined(__clang__) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_struct(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg(1, s, 2) != 2 * a + 3 * b + 9) + return 1; + + return 0; +} + +#endif + +#ifdef __SIZEOF_INT128__ + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg(1, 2, v) != 4 * a + 3 * b + 5) + return 1; + + return 0; +} + +/* + * arm64 rounds the register number up to an even one for an argument + * aligned to 16 bytes, so it wants this __int128 in x2 and x3, leaving x1 + * empty, and the last argument in x4. The x86-64 ABI has no such rule. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_odd(1, v, 2) != 1 + 2 * b + 3 * a + 4 * 2) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_last_regs(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_nofit(1, 2, 3, 4, s) != 5 * a + 6 * b + 30) + return 1; + + return 0; +} + +/* + * The x86-64 ABI moves an argument its six remaining registers cannot hold + * wholly onto the stack. arm64, with eight argument registers, still has a + * pair for it. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_straddle(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_big_arg s = { .a = a, .b = b }; + + if (bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s) != 6 * a + 7 * b + 55) + return 1; + + return 0; +} + +/* The same, with an argument after the struct to take the eighth register. */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_tail(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_tail(1, 2, 3, 4, 5, s, 6) != 6 * a + 7 * b + 103) + return 1; + + return 0; +} + +/* + * arm64 gives no register to an argument its eight registers cannot hold, + * nor to anything after it. Past its six registers the x86-64 ABI has both + * eightbytes on the stack either way. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_split8(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_split8(1, 2, 3, 4, 5, 6, 7, s) != 8 * a + 9 * b + 140) + return 1; + + return 0; +} + +#ifdef __SIZEOF_INT128__ + +/* + * The hole of aggregate_arg_kfunc_int128_odd, with arguments after it that + * arm64 shifts through x5 to x7, where its JIT keeps the BPF stack slots, + * and onto its stack. The second __int128 then leaves the second arm64 + * stack slot empty. x86-64 packs the eightbytes like the BPF convention, + * so nothing moves there. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd_many(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + u128 w = ((u128)~a << 64) | ~b; + + if (bpf_kfunc_call_test_i128_arg_odd_many(1, v, 2, 3, 4, 5, 6, w) != + 1 + 2 * b + 3 * a + 4 * 2 + 5 * 3 + 6 * 4 + 7 * 5 + 8 * 6 + + 9 * ~b + 10 * ~a) + return 1; + + return 0; +} + +/* + * x86-64 moves s to the stack and gives R9 back to the seventh argument. + * w, aligned to 16 bytes by the __int128 it holds, then leaves a stack slot + * empty before it: the fourth on x86-64, the second on arm64. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_stack_hole(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + struct prog_test_i128_arg w = { .v = ((u128)~a << 64) | ~b }; + + if (bpf_kfunc_call_test_i128_arg_stack_hole(1, 2, 3, 4, 5, s, 6, 7, w) != + 1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * a + 7 * b + 8 * 6 + 9 * 7 + + 10 * ~b + 11 * ~a) + return 1; + + return 0; +} + +/* + * Both conventions pad the stack to align this __int128, and the BPF + * convention pads for neither, so both JITs move it up an eightbyte. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_pad(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_pad(1, 2, 3, 4, 5, 6, 7, v) != 9 * a + 8 * b + 140) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R1 type STRUCT is not composed of scalars") +int aggregate_arg_kfunc_ptr_member(struct __sk_buff *skb) +{ + struct prog_test_ptr_arg s = { .p = skb, .x = 1 }; + + return bpf_kfunc_call_test_ptr_arg(s); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c new file mode 100644 index 0000000..f0077d9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c
@@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" + +typedef unsigned __int128 u128; + +__naked u128 global_agg_good(void) +{ + asm volatile ( + "r0 = 0x1234;" /* low 64 bits */ + "r2 = 0x5678;" /* high 64 bits */ + "exit;" + ); +} + +__naked u128 global_agg_bad(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked u128 global_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_global_fail(void) +{ + asm volatile ( + "call %[global_agg_bad];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__failure __msg("At subprogram exit the register R2 is not a scalar value") +__naked int aggregate_ret_global_ptr_fail(void) +{ + asm volatile ( + "call %[global_agg_bad_ptr];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" /* stack pointer placed in the second return register */ + "exit;" + ); +} + +/* + * R2 is a return register once the subprogram returns a pair, so a stack + * pointer left in it is rejected at the callee's exit exactly as one in R0 + * is: the callee frame is gone by the time the caller could use it. + */ +SEC("tc") +__load_if_JITed() +__failure __msg("cannot return stack pointer to the caller") +__naked int aggregate_ret_static_ptr_fail(void) +{ + asm volatile ( + "call %[static_agg_bad_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_no_r2(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_static_uninit_fail(void) +{ + asm volatile ( + "call %[static_agg_no_r2];" + "r0 = r2;" + "exit;" + : + : __imm(static_agg_no_r2) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_precise(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 4;" /* second half; its value is made precise below */ + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r2 stack= before 11: (b7) r2 = 4") +__naked int aggregate_ret_static_precise(void) +{ + asm volatile ( + "call %[static_agg_precise];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_precise) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 0: (85) call pc+9") +__naked int aggregate_ret_global_precise(void) +{ + asm volatile ( + "call %[global_agg_good];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(global_agg_good) + : __clobber_all); +} + +#if defined(__clang_major__) && __clang_major__ >= 23 + +/* A by-value struct that smuggles a pointer, which must be rejected. */ +struct with_ptr { + void *p; + __u64 x; +}; + +/* A by-value union that smuggles a pointer, which must be rejected too. */ +union upair_with_ptr { + void *p; + __u64 halves[2]; +}; + +__naked struct with_ptr global_ret_struct_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_struct_ptr() has unsupported return type") +__naked int aggregate_ret_global_struct_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_struct_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_struct_ptr) + : __clobber_all); +} + +__naked union upair_with_ptr global_ret_union_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_union_ptr() has unsupported return type") +__naked int aggregate_ret_global_union_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_union_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_union_ptr) + : __clobber_all); +} + +#endif + +/* + * gcc returns a by-value struct through a hidden pointer, and emits the + * 'r0 = r1' returning it after the __naked body's exit, leaving the + * subprogram falling through. Build these with clang only. + */ +#if defined(__clang__) + +struct arena_pair { + void __arena *lo; + void __arena *hi; +}; + +struct arena_and_scalar { + void __arena *p; + __u64 x; +}; + +struct arena_array { + void __arena *p[2]; +}; + +struct arena_single { + void __arena *p; +}; + +union arena_upair { + void __arena *p; + __u64 halves[2]; +}; + +__naked struct arena_pair global_ret_arena_pair(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_pair(void) +{ + asm volatile ( + "call %[global_ret_arena_pair];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_pair) + : __clobber_all); +} + +__naked struct arena_and_scalar global_ret_arena_and_scalar(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_and_scalar(void) +{ + asm volatile ( + "call %[global_ret_arena_and_scalar];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_and_scalar) + : __clobber_all); +} + +__naked struct arena_array global_ret_arena_array(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_array(void) +{ + asm volatile ( + "call %[global_ret_arena_array];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_array) + : __clobber_all); +} + +__naked struct arena_single global_ret_arena_single(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__success __retval(0) +__naked int aggregate_ret_global_arena_single(void) +{ + asm volatile ( + "call %[global_ret_arena_single];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_single) + : __clobber_all); +} + +__naked union arena_upair global_ret_arena_union(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_union(void) +{ + asm volatile ( + "call %[global_ret_arena_union];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_union) + : __clobber_all); +} + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c new file mode 100644 index 0000000..0361a7b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c
@@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Reference kfunc addresses to force those BTF to be emitted. Taking the address + * (rather than calling) avoids any dependence on the compiler lowering an + * __int128 or struct return value, which the BPF backend only supports from + * LLVM 23 on. + */ +void __kfunc_btf_root(void) +{ + asm volatile ("" + : + : "r"(&bpf_kfunc_call_test_i128), + "r"(&bpf_kfunc_call_test_ret_fastcall), + "r"(&bpf_kfunc_call_test_ret_ptr), + "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_nested), + "r"(&bpf_kfunc_call_test_ret_ptr_arr), + "r"(&bpf_kfunc_call_test_ret_deep), + "r"(&bpf_kfunc_call_test_ret_arr_struct), + "r"(&bpf_kfunc_call_test_ret_arr2d), + "r"(&bpf_kfunc_call_test_ret_big)); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 7 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 6: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 5: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 2: (85) call bpf_kfunc_call_test_i128") +__naked int aggregate_ret_kfunc_precise(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_i128];" + "r6 = r2;" /* second return half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_i128) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("kfunc bpf_kfunc_call_test_ret_fastcall with >8-byte return is not supported with KF_FASTCALL") +__naked int aggregate_ret_kfunc_fastcall_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_ret_fastcall];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_fastcall) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p' has type PTR") +__naked int aggregate_ret_kfunc_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in.p' has type PTR") +__naked int aggregate_ret_kfunc_nested_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_nested];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_nested) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p[]' has type PTR") +__naked int aggregate_ret_kfunc_ptr_arr_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_ptr_arr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr_arr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in1[].in2.p' has type PTR") +__naked int aggregate_ret_kfunc_arr_struct_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr_struct];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr_struct) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'a[].p' has type PTR") +__naked int aggregate_ret_kfunc_arr2d_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr2d];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr2d) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("max struct nesting depth exceeded") +__naked int aggregate_ret_kfunc_too_deep_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_deep];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_deep) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_kfunc_small_no_r2(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "call %[bpf_kfunc_call_test_ret_ii];" + "r0 = r2;" /* R2 is not a return register for a <=8 byte struct */ + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ii) + : __clobber_all); +} + +/* + * A return value larger than 16 bytes does not fit in R0:R2 and is rejected by + * btf_distill_func_proto() before the KF_FASTCALL and JIT-capability checks. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("The function bpf_kfunc_call_test_ret_big return type STRUCT is unsupported") +__naked int aggregate_ret_kfunc_too_big_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_big];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_big) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c new file mode 100644 index 0000000..a0f2636 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c
@@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena(void *ctx) +{ + u32 volatile __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *a, *b; + struct prog_test_ret_arena r; + + if (!page) + return 1; + + /* Both halves come back in R0:R2, pointing at page and page + 4. */ + r = bpf_kfunc_call_test_ret_arena((u64)page); + if (!r.a || !r.b) + return 2; + + a = (u32 __arena *)r.a; + b = (u32 __arena *)r.b; + *a = 1; + *b = 2; + if (*a != 1) + return 3; + if (*b != 2) + return 4; + + page[0] = 7; + if (*a != 7) + return 5; + page[1] = 9; + if (*b != 9) + return 6; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena_mixed(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + struct prog_test_ret_arena_mixed r; + u32 volatile __arena *p; + + if (!page) + return 1; + + /* An arena pointer in R0 beside a scalar in R2. */ + r = bpf_kfunc_call_test_ret_arena_mixed((u64)page); + if (!r.p) + return 2; + if (r.tag != 0xbeef) + return 3; + + p = (u32 __arena *)r.p; + *p = 3; + if (*p != 3) + return 4; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_untagged_fail(void *ctx) +{ + struct prog_test_ret_arena_untagged r; + + r = bpf_kfunc_call_test_ret_arena_untagged(0); + + return r.a == r.b; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_union_fail(void *ctx) +{ + union prog_test_ret_arena_union r; + + r = bpf_kfunc_call_test_ret_arena_union(0); + + return r.a == r.b; +} + +#else + +SEC("socket") +__description("aggregate_ret_kfunc_arena: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_target.c b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c new file mode 100644 index 0000000..cffd8d7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c
@@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* freplace target: a global subprogram returning 16 bytes in R0:R2. */ +__naked unsigned __int128 agg_ret_target_func(void) +{ + asm volatile ( + "r0 = 0x1234;" + "r2 = 0x5678;" + "exit;" + ); +} + +SEC("tc") +__naked int agg_ret_target(void) +{ + asm volatile ( + "call %[agg_ret_target_func];" + "r0 = 0;" + "exit;" + : + : __imm(agg_ret_target_func) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index 73bc2b8..ce76a73 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c
@@ -91,13 +91,17 @@ int sub(const void *ctx) return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u64 __arena_global and64_value = (0x110ull << 32); -_Atomic __u32 __arena_global and32_value = 0x110; -#else +/* + * A relaxed __c11_atomic_fetch_*() whose result is discarded is what selects + * the non-fetching insn, so each operation keeps a *_noreturn_value of each + * width to cover that lowering as well. + */ __u64 __arena_global and64_value = (0x110ull << 32); +__u64 __arena_global and64_result = 0; +_Atomic __u64 __arena_global and64_noreturn_value = (0x110ull << 32); __u32 __arena_global and32_value = 0x110; -#endif +__u32 __arena_global and32_result = 0; +_Atomic __u32 __arena_global and32_noreturn_value = 0x110; SEC("raw_tp/sys_enter") int and(const void *ctx) @@ -105,25 +109,22 @@ int and(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_and(&and64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_and(&and32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_and(&and64_value, 0x011ull << 32); - __sync_fetch_and_and(&and32_value, 0x011); -#endif + and64_result = __sync_fetch_and_and(&and64_value, 0x011ull << 32); + and32_result = __sync_fetch_and_and(&and32_value, 0x011); + __c11_atomic_fetch_and(&and64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_and(&and32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u32 __arena_global or32_value = 0x110; -_Atomic __u64 __arena_global or64_value = (0x110ull << 32); -#else +/* u32s first: .addr_space.1 must not have holes, see the clang-17 note below */ __u32 __arena_global or32_value = 0x110; +__u32 __arena_global or32_result = 0; +_Atomic __u32 __arena_global or32_noreturn_value = 0x110; __u64 __arena_global or64_value = (0x110ull << 32); -#endif +__u64 __arena_global or64_result = 0; +_Atomic __u64 __arena_global or64_noreturn_value = (0x110ull << 32); SEC("raw_tp/sys_enter") int or(const void *ctx) @@ -131,25 +132,21 @@ int or(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_or(&or64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_or(&or32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_or(&or64_value, 0x011ull << 32); - __sync_fetch_and_or(&or32_value, 0x011); -#endif + or64_result = __sync_fetch_and_or(&or64_value, 0x011ull << 32); + or32_result = __sync_fetch_and_or(&or32_value, 0x011); + __c11_atomic_fetch_or(&or64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_or(&or32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u64 __arena_global xor64_value = (0x110ull << 32); -_Atomic __u32 __arena_global xor32_value = 0x110; -#else __u64 __arena_global xor64_value = (0x110ull << 32); +__u64 __arena_global xor64_result = 0; +_Atomic __u64 __arena_global xor64_noreturn_value = (0x110ull << 32); __u32 __arena_global xor32_value = 0x110; -#endif +__u32 __arena_global xor32_result = 0; +_Atomic __u32 __arena_global xor32_noreturn_value = 0x110; SEC("raw_tp/sys_enter") int xor(const void *ctx) @@ -157,13 +154,10 @@ int xor(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_xor(&xor64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_xor(&xor32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_xor(&xor64_value, 0x011ull << 32); - __sync_fetch_and_xor(&xor32_value, 0x011); -#endif + xor64_result = __sync_fetch_and_xor(&xor64_value, 0x011ull << 32); + xor32_result = __sync_fetch_and_xor(&xor32_value, 0x011); + __c11_atomic_fetch_xor(&xor64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_xor(&xor32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; @@ -213,23 +207,96 @@ int xchg(const void *ctx) return 0; } +__u64 __arena_global fetch_src_r0_value = 0x11000000110ull; +__u64 __arena_global fetch_src_r0_result = 0; +__u64 __arena_global fetch_dst_r0_value = 0x11000000110ull; +__u64 __arena_global fetch_dst_r0_result = 0; +__u64 __arena_global fetch_dst_r0_readback = 0; + +/* + * A fetching OR with the operand in r0, and one with the arena pointer in r0. + * The x86 JIT needs RAX for its CMPXCHG loop and substitutes BPF_REG_AX for + * whichever of the two is r0, so both have to keep working. Hand-written + * because clang picks its own registers and will not reliably emit either. + */ +SEC("raw_tp/sys_enter") +int fetch_r0(const void *ctx) +{ + if (pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; +#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST) + asm volatile ( + "r1 = %[fetch_src_r0_value] ll;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r0 = 0x1100000011 ll;" + ".8byte %[fetch_src_r0_insn];" + "r2 = %[fetch_src_r0_result] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + : + : __imm_addr(fetch_src_r0_value), + __imm_insn(fetch_src_r0_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_1, BPF_REG_0, 0)), + __imm_addr(fetch_src_r0_result) + : __clobber_all); + + asm volatile ( + "r0 = %[fetch_dst_r0_value] ll;" + "r0 = addr_space_cast(r0, 0x0, 0x1);" + "r1 = 0x1100000011 ll;" + ".8byte %[fetch_dst_r0_insn];" + "r2 = %[fetch_dst_r0_result] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r1;" + /* r0 is only read by the atomic, so it must still be the pointer. */ + "r3 = *(u64 *)(r0 + 0);" + "r2 = %[fetch_dst_r0_readback] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r3;" + : + : __imm_addr(fetch_dst_r0_value), + __imm_insn(fetch_dst_r0_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)), + __imm_addr(fetch_dst_r0_result), + __imm_addr(fetch_dst_r0_readback) + : __clobber_all); +#endif + + return 0; +} + __u64 __arena_global uaf_sink; volatile __u64 __arena_global uaf_recovery_fails; +/* Register state after the r0 forms below fault, checked by the test */ +__u64 __arena_global uaf_r1_after_dst_r0; +__u64 __arena_global uaf_r0_before_dst_r0; +__u64 __arena_global uaf_r0_after_dst_r0; +__u64 __arena_global uaf_r0_after_src_r0; +/* What reading the faulted address again gives, see the end of uaf() */ +__u64 __arena_global uaf_reread; SEC("syscall") int uaf(const void *ctx) { if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; +/* + * __BPF_FEATURE_ADDR_SPACE_CAST is required by the asm below, which spells + * addr_space_cast textually. That and the mnemonic both arrived in LLVM 19. + */ #if defined(ENABLE_ATOMICS_TESTS) && !defined(__TARGET_ARCH_arm64) && \ - !defined(__TARGET_ARCH_x86) + defined(__BPF_FEATURE_ADDR_SPACE_CAST) __u32 __arena *page32; __u64 __arena *page64; void __arena *page; page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); bpf_arena_free_pages(&arena, page, 1); - uaf_recovery_fails = 24; + uaf_recovery_fails = 26; + uaf_r1_after_dst_r0 = 1; + uaf_r0_after_dst_r0 = 1; + uaf_r0_after_src_r0 = 1; + uaf_reread = 1; page32 = (__u32 __arena *)page; uaf_sink += __sync_fetch_and_add(page32, 1); @@ -282,6 +349,74 @@ int uaf(const void *ctx) uaf_recovery_fails -= 1; uaf_sink += __sync_lock_test_and_set(page64, 1); uaf_recovery_fails -= 1; + + /* + * The x86 JIT needs RAX for the CMPXCHG loop it lowers a fetching + * AND/OR/XOR into, and substitutes BPF_REG_AX for r0 when r0 is either + * operand. Only then does the fault fixup have to name that register, + * so spell both forms out: clang picks its own and never produces them. + * + * The freed page is the first of the arena, whose 32-bit address is 0, + * which would make "the address register was left alone" an empty + * check. Use the page after it, which is unmapped as well. + */ + asm volatile ( + "r0 = %[page];" + "r0 += 4096;" + "r0 = addr_space_cast(r0, 0x0, 0x1);" + "r1 = 1;" + "r2 = %[uaf_r0_before_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + ".8byte %[fetch_dst_r0];" + "r2 = %[uaf_r0_after_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + "r2 = %[uaf_r1_after_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r1;" + : : [page]"r"(page64), + __imm_insn(fetch_dst_r0, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)), + __imm_addr(uaf_r0_before_dst_r0), + __imm_addr(uaf_r0_after_dst_r0), + __imm_addr(uaf_r1_after_dst_r0) + : "r0", "r1", "r2", "memory"); + uaf_recovery_fails -= 1; + + asm volatile ( + "r1 = %[page];" + "r1 += 4096;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r0 = 1;" + ".8byte %[fetch_src_r0];" + "r2 = %[uaf_r0_after_src_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + : : [page]"r"(page64), + __imm_insn(fetch_src_r0, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_1, BPF_REG_0, 0)), + __imm_addr(uaf_r0_after_src_r0) + : "r0", "r1", "r2", "memory"); + uaf_recovery_fails -= 1; + + /* + * If a fault were recovered by the arena mapping its zeroed scratch + * page rather than by the JIT's exception table entry, the access above + * would have succeeded and written there. The page has to still be + * unmapped, so that the read faults and gives 0. + */ + asm volatile ( + "r1 = %[page];" + "r1 += 4096;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r3 = *(u64 *)(r1 + 0);" + "r2 = %[uaf_reread] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r3;" + : : [page]"r"(page64), + __imm_addr(uaf_reread) + : "r1", "r2", "r3", "memory"); #endif return 0;
diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index bf0d304..5d1cb1a 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c
@@ -192,20 +192,61 @@ int arena_arg_unpopulated(void *ctx) return 0; } +/* + * A subprogram whose only arena use is the argument of a kfunc needs the + * arena base too, the JIT rebases the argument on it. The address arrives + * as a number, so no cast marks the subprogram either. + */ +__noinline int arena_arg_from_subprog(unsigned long addr) +{ + return bpf_kfunc_arena_arg_test((u64 *)addr) != 41; +} + +/* + * A NULL constant for the __arena argument takes an early exit in the + * verifier; the call needs the arena base all the same, the JIT rebases + * the argument with a NULL test. + */ SEC("syscall") __arch_x86_64 __arch_arm64 -__failure __msg("arena pointer requires a program with an associated arena") -int arena_arg_no_arena(void *ctx) +__success __retval(0) +int arena_arg_null(void *ctx) { - bpf_kfunc_arena_arg_test((u64 *)1); - return 0; + return bpf_kfunc_arena_cap_nullable_test(NULL) != 0; } SEC("syscall") __arch_x86_64 __arch_arm64 -__failure __msg("is not a pointer to arena or scalar") +__success __retval(0) +int arena_arg_subprog(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (arena_arg_from_subprog((unsigned long)val)) + ret = 2; + else + ret = *val != 42 ? 3 : 0; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); + return ret; +#else + return 0; +#endif +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("R1 type=fp expected=arena, scalar") int arena_arg_bad_reg(void *ctx) { u64 buf = 0; @@ -228,13 +269,29 @@ int arena_arg_stack(void *ctx) bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); return 0; } + +#if defined(__clang__) +/* The struct takes two slots, so the arena pointer is the sixth. */ +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack_after_pair(void *ctx) +{ + struct prog_test_pair_arg s = { .lo = 1, .hi = 2 }; + + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_call_test_pair_arena_arg(1, 2, 3, s, (u64 *)1); + return 0; +} +#endif #else SEC("syscall") __arch_x86_64 __arch_arm64 __description("arena_arg_stack: not supported, dummy test") +__skip("arena_arg_stack: not supported") __success -int arena_arg_stack(void *ctx) +int dummy_test(void *ctx) { return 0; }
diff --git a/tools/testing/selftests/bpf/progs/arena_memcg.c b/tools/testing/selftests/bpf/progs/arena_memcg.c new file mode 100644 index 0000000..aff7375 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_memcg.c
@@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* + * Number of pages. Must cover ARENA_FAULT on the smallest page size + * (64M/4K = 16384) yet stay under the 4G arena limit on 64K pages + * (50000*64K = 3.2G). + */ + __uint(max_entries, 50000); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */ +#else + __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */ +#endif +} arena SEC(".maps"); + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c new file mode 100644 index 0000000..edbd40c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c
@@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Programs without an arena. They cannot share an object with an arena map, + * libbpf binds the arena of an object to every program in it. + */ + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__flag(BPF_F_ARENA_SCALAR) +__arch_x86_64 +__arch_arm64 +__description("arena_scalar: no arena, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_arena(void) +{ + asm volatile (" \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/arena_race.c b/tools/testing/selftests/bpf/progs/arena_race.c new file mode 100644 index 0000000..54dbd66 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_race.c
@@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> + +const volatile __u64 old_marker = 0x1111222233334444ULL; +const volatile __u64 new_marker = 0x5555666677778888ULL; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1024); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); +#else + __ulong(map_extra, 0x1ull << 44); +#endif +} arena SEC(".maps"); + +bool skip; +int page_idx; + +void __arena *ptr; +void __arena *realloc_ptr; +bool realloc_after_free; +__u64 current_marker; +__u64 marker_seq; + +int target_tid; +int pause_on_zap; +int zap_entered; +int release; +int timed_out; + +__u64 trigger_pid_tgid; +long trigger_syscall; +int deferred_free; +int worker_armed; +int worker_exited; + +static __always_inline void wait_for_release(void) +{ + while (!*(volatile int *)&release && can_loop) + ; + if (!*(volatile int *)&release) + timed_out = 1; +} + +SEC("syscall") +int alloc_old(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + char __arena *base = arena_base(&arena); + + realloc_ptr = NULL; + ptr = bpf_arena_alloc_pages(&arena, base + (page_idx + 1) * __PAGE_SIZE, 1, + NUMA_NO_NODE, 0); + if (!ptr) + return 1; + p = (__u64 __arena *)ptr; + *p = old_marker; + current_marker = old_marker; +#else + skip = true; +#endif + return 0; +} + +SEC("syscall") +int free_page(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + __u64 marker; + + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, ptr, 1); + if (!realloc_after_free) + return 0; + + marker = new_marker + ++marker_seq; + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (!realloc_ptr) + return 0; + ptr = realloc_ptr; + p = (__u64 __arena *)ptr; + *p = marker; + current_marker = marker; +#endif + return 0; +} + +SEC("syscall") +int try_realloc(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (realloc_ptr) { + ptr = realloc_ptr; + p = (__u64 __arena *)realloc_ptr; + *p = new_marker; + current_marker = new_marker; + } +#endif + return 0; +} + +SEC("tp_btf/sys_enter") +int BPF_PROG(deferred_free_prog, struct pt_regs *regs, long id) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + if (!deferred_free || bpf_get_current_pid_tgid() != trigger_pid_tgid || + id != trigger_syscall) + return 0; + + deferred_free = 0; + /* The worker can run on another CPU before the kfunc returns. */ + worker_armed = 1; + bpf_arena_free_pages(&arena, ptr, 1); +#endif + return 0; +} + +SEC("fentry/arena_free_worker") +int BPF_PROG(trace_free_worker, struct work_struct *work) +{ + if (worker_armed && !target_tid) + target_tid = (__u32)bpf_get_current_pid_tgid(); + return 0; +} + +SEC("fexit/arena_free_worker") +int BPF_PROG(trace_free_worker_ret, struct work_struct *work) +{ + if ((__u32)bpf_get_current_pid_tgid() == target_tid) + worker_exited = 1; + return 0; +} + +/* + * Pause a free while it zaps user mappings: after it cleared the kernel PTEs + * and flushed the TLB, but before it releases the range. zap_vma_range() is + * generic MM code, so it is an fentry target even where + * flush_tlb_kernel_range() is inline (arm64). + */ +SEC("?fentry/zap_vma_range") +int BPF_PROG(trace_zap, struct vm_area_struct *vma, unsigned long address, + unsigned long size) +{ + if ((__u32)bpf_get_current_pid_tgid() != target_tid) + return 0; + zap_entered = 1; + if (pause_on_zap) + wait_for_release(); + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/async_stack_depth.c b/tools/testing/selftests/bpf/progs/async_stack_depth.c index 3673468..9cd874a 100644 --- a/tools/testing/selftests/bpf/progs/async_stack_depth.c +++ b/tools/testing/selftests/bpf/progs/async_stack_depth.c
@@ -29,7 +29,49 @@ static int bad_timer_cb(void *map, int *key, struct bpf_timer *timer) return buf[255] + timer_cb(NULL, NULL, NULL); } +/* + * The same shapes scaled to the 2 KiB budget of JITs with large stacks. The + * compiler caps a single function at 512 bytes, so the depth comes from a + * chain of 480-byte frames. + */ +__attribute__((noinline)) +static int timer_cb_large_0(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69]; +} + +__attribute__((noinline)) +static int timer_cb_large_1(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_0(map, key, timer); +} + +__attribute__((noinline)) +static int timer_cb_large_2(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_1(map, key, timer); +} + +__attribute__((noinline)) +static int timer_cb_large_3(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_2(map, key, timer); +} + +/* 5 * 480 = 2400 bytes on its own */ +__attribute__((noinline)) +static int bad_timer_cb_large(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[255] + timer_cb_large_3(map, key, timer); +} + SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 2 calls is") int pseudo_call_check(struct __sk_buff *ctx) { @@ -44,7 +86,25 @@ int pseudo_call_check(struct __sk_buff *ctx) return bpf_timer_set_callback(&elem->timer, timer_cb) + buf[0]; } +/* main plus the four frames under timer_cb_large_3: 2400 bytes */ SEC("tc") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is") +int pseudo_call_check_large(struct __sk_buff *ctx) +{ + struct hmap_elem *elem; + volatile char buf[480] = {}; + + elem = bpf_map_lookup_elem(&hmap, &(int){0}); + if (!elem) + return 0; + + timer_cb_large_3(NULL, NULL, NULL); + return bpf_timer_set_callback(&elem->timer, timer_cb_large_3) + buf[0]; +} + +SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 2 calls is") int async_call_root_check(struct __sk_buff *ctx) { @@ -58,4 +118,19 @@ int async_call_root_check(struct __sk_buff *ctx) return bpf_timer_set_callback(&elem->timer, bad_timer_cb) + buf[0]; } +SEC("tc") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is") +int async_call_root_check_large(struct __sk_buff *ctx) +{ + struct hmap_elem *elem; + volatile char buf[480] = {}; + + elem = bpf_map_lookup_elem(&hmap, &(int){0}); + if (!elem) + return 0; + + return bpf_timer_set_callback(&elem->timer, bad_timer_cb_large) + buf[0]; +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c b/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c index 00b2cea..b9107d3a 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c
@@ -25,12 +25,12 @@ int dump_netlink(struct bpf_iter__netlink *ctx) return 0; if (ctx->meta->seq_num == 0) - BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups " + BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups " "Rmem Wmem Dump Locks Drops " "Inode\n"); s = &nlk->sk; - BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol); + BPF_SEQ_PRINTF(seq, "0 %-3d ", s->sk_protocol); if (!nlk->groups) { group = 0;
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c index b1e509b2..d149cf0 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c
@@ -131,8 +131,7 @@ static int dump_tcp_sock(struct seq_file *seq, struct tcp_sock *tp, icsk->icsk_probes_out, sock_i_ino(sp), sp->sk_refcnt.refs.counter); - BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n", - tp, + BPF_SEQ_PRINTF(seq, "0 %lu %lu %u %u %d\n", jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk), @@ -161,10 +160,10 @@ static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw, BPF_SEQ_PRINTF(seq, "%4d: %08X:%04X %08X:%04X ", seq_num, src, srcp, dest, destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", tw->tw_substate, 0, 0, 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, - tw->tw_refcnt.refs.counter, tw); + tw->tw_refcnt.refs.counter); return 0; } @@ -185,9 +184,9 @@ static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq, seq_num, irsk->ir_loc_addr, irsk->ir_num, irsk->ir_rmt_addr, bpf_ntohs(irsk->ir_rmt_port)); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd), - req->num_timeout, uid, 0, 0, 0, req); + req->num_timeout, uid, 0, 0, 0); return 0; }
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c index dbc7166..cd65fa8 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c
@@ -135,8 +135,7 @@ static int dump_tcp6_sock(struct seq_file *seq, struct tcp6_sock *tp, icsk->icsk_probes_out, sock_i_ino(sp), sp->sk_refcnt.refs.counter); - BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n", - tp, + BPF_SEQ_PRINTF(seq, "0 %lu %lu %u %u %d\n", jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk), @@ -170,10 +169,10 @@ static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw, dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", tw->tw_substate, 0, 0, 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, - tw->tw_refcnt.refs.counter, tw); + tw->tw_refcnt.refs.counter); return 0; } @@ -201,9 +200,9 @@ static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq, dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], bpf_ntohs(irsk->ir_rmt_port)); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd), - req->num_timeout, uid, 0, 0, 0, req); + req->num_timeout, uid, 0, 0, 0); return 0; }
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c index 23b2aa2..d4079a6 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c
@@ -57,13 +57,13 @@ int dump_udp4(struct bpf_iter__udp *ctx) BPF_SEQ_PRINTF(seq, "%5d: %08X:%04X %08X:%04X ", ctx->bucket, src, srcp, dest, destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d 0 %u\n", inet->sk.sk_state, inet->sk.sk_wmem_alloc.refs.counter - 1, rqueue, 0, 0L, 0, ctx->uid, 0, sock_i_ino(&inet->sk), - inet->sk.sk_refcnt.refs.counter, udp_sk, + inet->sk.sk_refcnt.refs.counter, udp_sk->drop_counters.drops0.counter + udp_sk->drop_counters.drops1.counter);
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c index c48b05a..dadc5f7 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c
@@ -65,13 +65,13 @@ int dump_udp6(struct bpf_iter__udp *ctx) dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d 0 %u\n", inet->sk.sk_state, inet->sk.sk_wmem_alloc.refs.counter - 1, rqueue, 0, 0L, 0, ctx->uid, 0, sock_i_ino(&inet->sk), - inet->sk.sk_refcnt.refs.counter, udp_sk, + inet->sk.sk_refcnt.refs.counter, udp_sk->drop_counters.drops0.counter + udp_sk->drop_counters.drops1.counter); return 0;
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c index a2652c8..3061b14 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c
@@ -42,10 +42,9 @@ int dump_unix(struct bpf_iter__unix *ctx) seq = ctx->meta->seq; seq_num = ctx->meta->seq_num; if (seq_num == 0) - BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n"); + BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n"); - BPF_SEQ_PRINTF(seq, "%pK: %08X %08X %08X %04X %02X %8lu", - unix_sk, + BPF_SEQ_PRINTF(seq, "%08u: %08X %08X %08X %04X %02X %8lu", 0, sk->sk_refcnt.refs.counter, 0, sk->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0, @@ -71,7 +70,7 @@ int dump_unix(struct bpf_iter__unix *ctx) for (i = 1; i < len; i++) { /* unix_validate_addr() tests this upper bound. */ - if (i >= sizeof(struct sockaddr_un)) + if (i >= sizeof(unix_sk->addr->name->sun_path)) break; BPF_SEQ_PRINTF(seq, "%c",
diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 5eacf1b..d7f706c 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h
@@ -9,7 +9,8 @@ #define QUOTE(str) #str #define EXPAND_QUOTE(str) QUOTE(str) -/* This set of attributes controls behavior of the +/* + * This set of attributes controls behavior of the * test_loader.c:test_loader__run_subtests(). * * The test_loader sequentially loads each program in a skeleton. @@ -42,6 +43,10 @@ * __msg_unpriv Same as __msg but for unprivileged mode. * __not_msg_unpriv Same as __not_msg but for unprivileged mode. * + * __msg_next Same as __msg but expects the match to be on the next + * __msg_next_unpriv log line compared to the last match, not just some line + * after the last match. + * * __stderr Message expected to be found in bpf stderr stream. The * same regex rules apply like __msg. * __stderr_unpriv Same as __stderr but for unpriveleged mode. @@ -106,6 +111,11 @@ * __description Text to be used for display and as an additional filter * alias, while the original program name stays matchable. * + * __skip Report the test as SKIP with the given reason instead of + * running it. For placeholder programs that stand in for a + * feature the toolchain or JIT cannot provide, so a run does + * not report OK for coverage it never executed. + * * __log_level Log level to use for the program, numeric value expected. * * __flag Adds one flag use for the program, the following values are valid: @@ -126,6 +136,11 @@ * Several __arch_* annotations could be specified at once. * When test case is not run on current arch it is marked as skipped. * __caps_unpriv Specify the capabilities that should be set when running the test. + * __prepare_priv In unprivileged mode, prepare the object with the fixture's + * initial capabilities before dropping them for program loading. + * Preparation includes map creation and BTF/kfunc resolution; + * these operations are not tested at the reduced capabilities. + * Program loading uses the normal __caps_unpriv selection. * * __linear_size Specify the size of the linear area of non-linear skbs, or * 0 for linear skbs. @@ -134,13 +149,16 @@ #define __msg(msg) __test_tag("test_expect_msg=" msg) #define __not_msg(msg) __test_tag("test_expect_not_msg=" msg) +#define __msg_next(msg) __test_tag("test_expect_msg_next=" msg) #define __xlated(msg) __test_tag("test_expect_xlated=" msg) #define __jited(msg) __test_tag("test_jited=" msg) #define __failure __test_tag("test_expect_failure") #define __success __test_tag("test_expect_success") #define __description(desc) __test_tag("test_description=" desc) +#define __skip(reason) __test_tag("test_skip=" reason) #define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg) #define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg) +#define __msg_next_unpriv(msg) __test_tag("test_expect_msg_next_unpriv=" msg) #define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg) #define __jited_unpriv(msg) __test_tag("test_jited_unpriv=" msg) #define __failure_unpriv __test_tag("test_expect_failure_unpriv") @@ -160,8 +178,12 @@ #define __arch_s390x __arch("s390x") #define __arch_loongarch __arch("LOONGARCH") #define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps)) +#define __prepare_priv __test_tag("test_prepare_priv") #define __load_if_JITed() __test_tag("load_mode=jited") #define __load_if_no_JITed() __test_tag("load_mode=no_jited") +/* Whether programs may use more than 512 bytes of stack on this kernel and JIT */ +#define __load_if_large_stack() __test_tag("stack_mode=large") +#define __load_if_no_large_stack() __test_tag("stack_mode=small") #define __stderr(msg) __test_tag("test_expect_stderr=" msg) #define __stderr_unpriv(msg) __test_tag("test_expect_stderr_unpriv=" msg) #define __stdout(msg) __test_tag("test_expect_stdout=" msg) @@ -255,11 +277,11 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_GOTOL)) #define CAN_USE_GOTOL #endif -#if __clang_major__ >= 18 +#if __clang_major__ >= 18 || defined(__BPF_FEATURE_ST) #define CAN_USE_BPF_ST #endif
diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c new file mode 100644 index 0000000..8217f4c4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c
@@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +static __noinline struct ethhdr *slice_in_subprog(struct sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + return bpf_dynptr_slice(&ptr, 0, NULL, sizeof(struct ethhdr)); +} + +SEC("struct_ops") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(invalid_dynptr_returned_slice, struct sk_buff *skb, + struct Qdisc *sch, struct bpf_sk_buff_ptr *to_free) +{ + struct ethhdr *hdr; + + hdr = slice_in_subprog(skb); + if (!hdr) { + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; + } + + /* this should fail */ + proto = hdr->h_proto; + + bpf_qdisc_skb_drop(skb, to_free); + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)invalid_dynptr_returned_slice, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +};
diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c new file mode 100644 index 0000000..94a7f52 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c
@@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define MAX_CGROUP_OPS 8 + +/* Call order for listen and connect, indexed by call sequence */ +u32 listen_order[MAX_CGROUP_OPS]; +u32 listen_cnt; + +u32 connect_order[MAX_CGROUP_OPS]; +u32 connect_cnt; + +static void record_listen(int id) +{ + u32 idx = listen_cnt; + + if (idx < MAX_CGROUP_OPS) { + listen_order[idx] = id; + listen_cnt = idx + 1; + } +} + +static void record_connect(int id) +{ + u32 idx = connect_cnt; + + if (idx < MAX_CGROUP_OPS) { + connect_order[idx] = id; + connect_cnt = idx + 1; + } +} + +/* struct_ops instance 1 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops1_listen, struct sock *sk) +{ + record_listen(1); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops1_connect, struct sock *sk) +{ + record_connect(1); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops1 = { + .listen = (void *)tcp_ops1_listen, + .connect = (void *)tcp_ops1_connect, +}; + +/* struct_ops instance 2 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops2_listen, struct sock *sk) +{ + record_listen(2); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops2_connect, struct sock *sk) +{ + record_connect(2); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops2 = { + .listen = (void *)tcp_ops2_listen, + .connect = (void *)tcp_ops2_connect, +}; + +/* struct_ops instance 3 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops3_listen, struct sock *sk) +{ + record_listen(3); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops3_connect, struct sock *sk) +{ + record_connect(3); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops3 = { + .listen = (void *)tcp_ops3_listen, + .connect = (void *)tcp_ops3_connect, +}; + +#define OPS_RETVAL1 11 +#define OPS_RETVAL2 22 +#define OPS_RETVAL3 33 + +int retval_saw1; +int retval_saw2; +int retval_saw3; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval1_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw1 = bpf_get_retval(); + return OPS_RETVAL1; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval1 = { + .timeout_init = (void *)tcp_ops_retval1_timeout_init, +}; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval2_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw2 = bpf_get_retval(); + return OPS_RETVAL2; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval2 = { + .timeout_init = (void *)tcp_ops_retval2_timeout_init, +}; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval3_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw3 = bpf_get_retval(); + return OPS_RETVAL3; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval3 = { + .timeout_init = (void *)tcp_ops_retval3_timeout_init, +}; + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c new file mode 100644 index 0000000..706e94d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c
@@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +/* Experimental option kind and payload written/parsed by this test. */ +#define TEST_OPT_KIND 0xFD +#define TEST_OPT_LEN 4 +#define TEST_OPT_D0 0xAB +#define TEST_OPT_D1 0xCD + +int hdr_opt_len_cnt; +int write_cnt; +int parse_cnt; +int found_cnt; +__u8 found_d0; +__u8 found_d1; + +SEC("struct_ops") +void BPF_PROG(test_listen, struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL | + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT, + 0); +} + +SEC("struct_ops") +void BPF_PROG(test_connect, struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN | + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT, + 0); +} + +SEC("struct_ops") +void BPF_PROG(test_hdr_opt_len, struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, unsigned int *remaining) +{ + hdr_opt_len_cnt++; + + /* + * Reserve TEST_OPT_LEN bytes; the helper decrements *remaining. Stacks + * with other progs in the cgroup hierarchy. + */ + bpf_reserve_hdr_opt(ctx, TEST_OPT_LEN, 0); +} + +SEC("struct_ops") +void BPF_PROG(test_write_hdr_opt, struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, __u32 opt_off) +{ + __u8 opt[TEST_OPT_LEN] = { + TEST_OPT_KIND, TEST_OPT_LEN, TEST_OPT_D0, TEST_OPT_D1, + }; + + /* + * bpf_store_hdr_opt() takes the program ctx (the kernel reads the + * outgoing skb from it); it appends after any options already written + * in the reserved window, rejects duplicates, and confines the write to + * the header option scratch. Stacks across progs in the cgroup hierarchy. + */ + if (bpf_store_hdr_opt(ctx, opt, sizeof(opt), 0)) + return; + + write_cnt++; +} + +SEC("struct_ops") +void BPF_PROG(test_parse_hdr, struct sock *sk, struct sk_buff *skb) +{ + __u8 opt[TEST_OPT_LEN] = { + TEST_OPT_KIND, TEST_OPT_LEN, TEST_OPT_D0, TEST_OPT_D1, + }; + + parse_cnt++; + + /* + * Look up the experimental option written by test_write_hdr_opt() in + * the incoming skb. For an experimental kind the search matches on the + * 2-byte magic in opt[2..3]; on a match the found option is copied back + * into opt[]. + */ + if (bpf_load_hdr_opt(ctx, opt, sizeof(opt), 0) < 0) + return; + + found_d0 = opt[2]; + found_d1 = opt[3]; + found_cnt++; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops test_hdr_ops = { + .listen = (void *)test_listen, + .connect = (void *)test_connect, + .hdr_opt_len = (void *)test_hdr_opt_len, + .write_hdr_opt = (void *)test_write_hdr_opt, + .parse_hdr = (void *)test_parse_hdr, +}; + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index 593b38f..4c999d5 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
@@ -79,6 +79,8 @@ #define NEXTHDR_TCP 6 +#define TCPHDR_FIN 0x01 + #define TCPOPT_NOP 1 #define TCPOPT_EOL 0 #define TCPOPT_MSS 2
diff --git a/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c b/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c new file mode 100644 index 0000000..bbbc93c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c
@@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, 1); +} perfbuf SEC(".maps"); + +/* + * Include the u32 raw size in perf's 8-byte alignment to avoid padding. + * Keep plain output large enough to flush stdio before the consumer exits. + */ +const unsigned char perf_payload[4092] = { 0x00, 0xff }; + +int record; +int output_err; + +SEC("socket") +int produce(struct __sk_buff *skb) +{ + unsigned char first[] = { 0x00, 0xff }; + unsigned char second[] = { 1, 2, 3, 4, 5 }; + unsigned char third[] = { + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + }; + + if (record == 0) + output_err = bpf_ringbuf_output(&ringbuf, first, sizeof(first), 0); + else if (record == 1) + output_err = bpf_ringbuf_output(&ringbuf, second, sizeof(second), 0); + else if (record == 2) + output_err = bpf_ringbuf_output(&ringbuf, third, sizeof(third), 0); + else + output_err = bpf_perf_event_output(skb, &perfbuf, BPF_F_CURRENT_CPU, + (void *)perf_payload, sizeof(perf_payload)); + return 0; +} + +char LICENSE[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c index 8d38aaf..ab12397a 100644 --- a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c
@@ -4,7 +4,8 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) long subprog_call_mem_kfunc(long a, long b, long c, long d, long e, long size)
diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index 99bc115..a980a3a 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c
@@ -3,7 +3,8 @@ #include <vmlinux.h> #include <bpf/bpf_helpers.h> -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) int subprog_bad_order_6args(int a, int b, int c, int d, int e, int f)
diff --git a/tools/testing/selftests/bpf/progs/btf_module_allowlist.c b/tools/testing/selftests/bpf/progs/btf_module_allowlist.c new file mode 100644 index 0000000..bb3c38b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf_module_allowlist.c
@@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +SEC("fentry/bpf_testmod_loop_test") +int BPF_PROG(test_btf_module_allowlist) +{ + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/call_rcu.c b/tools/testing/selftests/bpf/progs/call_rcu.c new file mode 100644 index 0000000..5050086 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/call_rcu.c
@@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +/* rh is not at offset 0, so the callback's value recovery is exercised. */ +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, __u32); + __type(value, struct elem); +} arr SEC(".maps"); + +__u32 cb_key; +__u64 cb_keys; +__u64 cb_val; +__u32 cb_max_entries; +int callbacks; +int arm_err; +int busy_err; +int chain; /* set by userspace: number of times to re-arm from the callback */ +int chain_err; + +static int reclaim(struct bpf_map *map, void *key, void *value) +{ + struct elem *e = value; + + cb_key = *(__u32 *)key; + __sync_fetch_and_or(&cb_keys, 1ULL << cb_key); + cb_val = e->val; + cb_max_entries = map->max_entries; + e->val = 0; + + if (chain > 0) { + chain--; + chain_err = bpf_call_rcu(&e->rh, &arr, reclaim); + } + + __sync_fetch_and_add(&callbacks, 1); + return 0; +} + +SEC("syscall") +int arm(void *ctx) +{ + __u32 key = 1; + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 1; + + e->val = 0xdeadbeef; + /* Keep a grace period from elapsing between the two arms. */ + bpf_rcu_read_lock(); + arm_err = bpf_call_rcu(&e->rh, &arr, reclaim); + busy_err = bpf_call_rcu(&e->rh, &arr, reclaim); + bpf_rcu_read_unlock(); + return 0; +} + +SEC("syscall") +int arm_both(void *ctx) +{ + __u32 key0 = 0, key1 = 1; + struct elem *e0, *e1; + + e0 = bpf_map_lookup_elem(&arr, &key0); + e1 = bpf_map_lookup_elem(&arr, &key1); + if (!e0 || !e1) + return 1; + + e0->val = 0xdeadbeef; + e1->val = 0xdeadbeef; + arm_err = bpf_call_rcu(&e0->rh, &arr, reclaim); + arm_err |= bpf_call_rcu(&e1->rh, &arr, reclaim); + return 0; +} + +SEC("syscall") +int arm_trace(void *ctx) +{ + __u32 key = 1; + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 1; + + e->val = 0xdeadbeef; + /* No lock needed: the enclosing rcu_read_lock_trace() already blocks the grace period. */ + arm_err = bpf_call_rcu_tasks_trace(&e->rh, &arr, reclaim); + busy_err = bpf_call_rcu_tasks_trace(&e->rh, &arr, reclaim); + return 0; +} + +SEC("iter/bpf_map_elem") +int dump(struct bpf_iter__bpf_map_elem *ctx) +{ + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/call_rcu_fail.c b/tools/testing/selftests/bpf/progs/call_rcu_fail.c new file mode 100644 index 0000000..bf60a27 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/call_rcu_fail.c
@@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +const void *user_ptr = NULL; + +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct elem); +} arr SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct elem); +} arr2 SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} plain SEC(".maps"); + +__u32 key = 0; + +static int reclaim(struct bpf_map *map, void *key, void *value) +{ + return 0; +} + +static int sleepable_reclaim(struct bpf_map *map, void *key, void *value) +{ + struct elem *e = value; + + bpf_copy_from_user(&e->val, sizeof(e->val), user_ptr); + return 0; +} + +SEC("syscall") +__failure __msg("bpf_rcu_head pointer in R1") __msg("doesn't match map pointer in R2") +int mismatch_map(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu(&e->rh, &arr2, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("map 'plain' has no valid bpf_rcu_head") +int no_rcu_head(void *ctx) +{ + __u64 *val; + + val = bpf_map_lookup_elem(&plain, &key); + if (!val) + return 0; + bpf_call_rcu((struct bpf_rcu_head *)val, &plain, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("doesn't point to 'struct bpf_rcu_head' that is at 8") +int wrong_offset(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu((struct bpf_rcu_head *)&e->pad, &arr, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("R1 type=fp expected=map_value") +int rcu_head_on_stack(void *ctx) +{ + struct bpf_rcu_head rh; + + bpf_call_rcu(&rh, &arr, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_copy_from_user") __msg("in non-sleepable prog") +int sleepable_callback(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu(&e->rh, &arr, sleepable_reclaim); + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/callx_rodata.c b/tools/testing/selftests/bpf/progs/callx_rodata.c new file mode 100644 index 0000000..e3f8335 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/callx_rodata.c
@@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +/* callx through pointers to functions in .rodata, loaded by light skeleton */ + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* set by user space before the programs are loaded, it's in .rodata too */ +const volatile int bias = 1; + +int op_idx; + +#ifdef __clang__ + +const volatile bool skip = false; + +typedef int (*op_fn)(int); + +static __noinline int add_bias(int x) +{ + return x + bias; +} + +static __noinline int mul3(int x) +{ + return x * 3; +} + +/* volatile, so that the compiler doesn't replace the table with direct calls */ +static op_fn const volatile ops[] = { add_bias, mul3 }; + +SEC("socket") +int select_op(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + return ops[i](10); +} + +/* functions have other offsets in this program */ +SEC("socket") +int both_ops(void *ctx) +{ + return ops[1](ops[0](4)); +} + +#else + +/* gcc doesn't support indirect calls */ +const volatile bool skip = true; + +SEC("socket") +int select_op(void *ctx) +{ + return 0; +} + +SEC("socket") +int both_ops(void *ctx) +{ + return 0; +} + +#endif + +/* no callx here */ +SEC("socket") +int read_bias(void *ctx) +{ + return bias; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/cb_refs.c b/tools/testing/selftests/bpf/progs/cb_refs.c index 5d6fc7f..d4a691d 100644 --- a/tools/testing/selftests/bpf/progs/cb_refs.c +++ b/tools/testing/selftests/bpf/progs/cb_refs.c
@@ -2,6 +2,7 @@ #include <vmlinux.h> #include <bpf/bpf_tracing.h> #include <bpf/bpf_helpers.h> +#include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" struct map_value { @@ -24,6 +25,7 @@ static __noinline int cb1(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") int underflow_prog(void *ctx) { struct prog_test_ref_kfunc *p; @@ -47,6 +49,7 @@ static __always_inline int cb2(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=3") /* alloc_insn=3{0,2,3} */ int leak_prog(void *ctx) { struct prog_test_ref_kfunc *p; @@ -83,6 +86,7 @@ static __always_inline int cb3(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=2") /* alloc_insn=2{2,4,5} */ int nested_cb(void *ctx) { struct prog_test_ref_kfunc *p; @@ -98,6 +102,7 @@ int nested_cb(void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=1") /* alloc_insn=1{1,2} */ int non_cb_transfer_ref(void *ctx) { struct prog_test_ref_kfunc *p;
diff --git a/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.h similarity index 100% rename from tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h rename to tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.h
diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index efe7bca..ede6a17 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c
@@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("R1 is fp expected STRUCT cgroup") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; @@ -154,7 +154,7 @@ int BPF_PROG(cgrp_kfunc_xchg_unreleased, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_rcu_get_release, struct cgroup *cgrp, const char *path) { struct cgroup *kptr; @@ -191,7 +191,7 @@ int BPF_PROG(cgrp_kfunc_release_untrusted, struct cgroup *cgrp, const char *path } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(cgrp_kfunc_release_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired = (struct cgroup *)&path; @@ -237,7 +237,7 @@ int BPF_PROG(cgrp_kfunc_release_null, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_release_unacquired, struct cgroup *cgrp, const char *path) { /* Cannot release trusted cgroup pointer which was not acquired. */
diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index d055fc7..1ff24bb 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c
@@ -299,7 +299,15 @@ __naked void gotol(void) "r3 = 24;" "if r1 > 0x7 goto +2;" "r0 = r2;" +#ifdef __clang__ "gotol +1;" +#else + /* + * gas mis-parses 'gotol +1' as 'goto l+1', same encoding + * without the sign (binutils PR gas/34558). + */ + "gotol 1;" +#endif "r0 = r3;" "exit;" : @@ -406,13 +414,42 @@ static __used __naked int aux1(void) ::: __clobber_all); } +static __noinline __used int static_two(int a, int b) +{ + return a; +} + +__noinline __used int global_two(int a, int b) +{ + return a; +} + SEC("socket") __log_level(2) -__msg("0: ....45.... (b7) r1 = 1") -__msg("1: .1..45.... (b7) r2 = 2") -__msg("2: .12.45.... (b7) r3 = 3") -/* Conservative liveness for subprog parameters. */ -__msg("3: .12345.... (85) call pc+2") +/* a static callee reads r1 only, a global callee its declared arguments */ +__msg("2: .1........ (85) call pc+") +__msg("5: .12....... (85) call pc+") +__naked void subprog_declared_args(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call static_two;" + "r1 = 1;" + "r2 = 2;" + "call global_two;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__log_level(2) +__msg("0: .......... (b7) r1 = 1") +__msg("1: .1........ (b7) r2 = 2") +__msg("2: .12....... (b7) r3 = 3") +/* A static callee reads what its code reads: aux1 reads r1 and r2. */ +__msg("3: .12....... (85) call pc+2") __msg("4: .......... (b7) r0 = 0") __msg("5: 0......... (95) exit") __msg("6: .12....... (bf) r0 = r1") @@ -431,7 +468,36 @@ __naked void subprog1(void) ::: __clobber_all); } -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +static __used __naked unsigned __int128 aux2(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +/* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */ +__load_if_JITed() +__log_level(2) +__msg("0: .......... (85) call pc+2") +__msg("1: ..2....... (bf) r0 = r2") +__msg("2: 0......... (95) exit") +__msg("3: .......... (b7) r0 = 1") +__msg("4: 0......... (b7) r2 = 2") +__msg("5: 0.2....... (95) exit") +__naked void subprog_ret_reg_pair(void) +{ + asm volatile ( + "call aux2;" + "r0 = r2;" + "exit;" + ::: __clobber_all); +} + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) SEC("socket") __log_level(2) @@ -470,7 +536,7 @@ void gotox(void) : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 */ +#endif /* gotox: x86, arm64, riscv64 */ /* to retain debug info for BTF generation */ void kfunc_root(void)
diff --git a/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c new file mode 100644 index 0000000..00ecba3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c
@@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <errno.h> + +char _license[] SEC("license") = "GPL"; + +int monitored_pid; +int bprm_argc; +int bprm_envc; +int data_len; +int invalid_flags_ret; +int copy_ret; +int arg0_ret; +int arg1_ret; +int env0_ret; +int env1_ret; +char data[64] = {}; +char arg0[32] = {}; +char arg1[32] = {}; +char env0[32] = {}; +char env1[32] = {}; + +SEC("lsm.s/bprm_check_security") +int BPF_PROG(check_exec_args, struct linux_binprm *bprm) +{ + u32 pid = bpf_get_current_pid_tgid() >> 32; + struct mm_struct *mm; + u64 offset = 0; + + if (pid != monitored_pid) + return 0; + + mm = bprm->mm; + if (!mm) + return 0; + + bprm_argc = bprm->argc; + bprm_envc = bprm->envc; + + /* this is the total size of args and envs starting from bprm->p */ + data_len = bprm->exec - bprm->p; + + invalid_flags_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, mm, ~0ULL); + + copy_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, mm, 0); + if (copy_ret) + return 0; + + /* arg0 is at bprm->p */ + arg0_ret = bpf_copy_from_user_mm_str(arg0, sizeof(arg0), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += arg0_ret; + + /* arg1 is at bprm->p + arg0_ret */ + arg1_ret = bpf_copy_from_user_mm_str(arg1, sizeof(arg1), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += arg1_ret; + + /* env0 is at bprm->p + arg0_ret + arg1_ret */ + env0_ret = bpf_copy_from_user_mm_str(env0, sizeof(env0), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += env0_ret; + + /* env1 is at bprm->p + arg0_ret + arg1_ret + env0_ret */ + env1_ret = bpf_copy_from_user_mm_str(env1, sizeof(env1), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + + return -EPERM; +}
diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 4628feb..c89c88d 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c
@@ -183,7 +183,7 @@ int BPF_PROG(test_global_mask_no_null_check, struct task_struct *task, u64 clone } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to helper R2") +__failure __msg("release function bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int BPF_PROG(test_global_mask_rcu_no_null_check, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *prev, *curr; @@ -243,7 +243,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl } SEC("tp_btf/task_newtask") -__failure __msg("leads to invalid memory access") +__failure __msg("R2 type=scalar expected=fp") int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_flags) { void *garbage = (void *)0x123456;
diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 36f77b9..ddaba43 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c
@@ -460,6 +460,9 @@ int BPF_PROG(test_insert_leave, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *cpumask; + if (!is_test_task()) + return 0; + cpumask = create_cpumask(); if (!cpumask) return 0; @@ -476,6 +479,9 @@ int BPF_PROG(test_insert_remove_release, struct task_struct *task, u64 clone_fla struct bpf_cpumask *cpumask; struct __cpumask_map_value *v; + if (!is_test_task()) + return 0; + cpumask = create_cpumask(); if (!cpumask) return 0;
diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c new file mode 100644 index 0000000..18f3810 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena.c
@@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long pair[2] = { 1, 2 }; +int x = 42; +long sum; +long table[4]; +struct { + long v[8]; +} aligned64 __attribute__((aligned(64))); +const volatile int ro = 7; +const volatile long ro_table[4] = { 10, 20, 30, 40 }; + +/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ +const char hello[] SEC(".rodata.str.hello") = "hello"; + +/* pointers to data that are stored in data */ +int *px = &x; +const char *str = hello; +int *const volatile cpx SEC(".data.rel.ro") = &x; + +SEC("syscall") +int use_data(void *ctx) +{ + int i; + + for (i = 0; i < 4; i++) + table[i] = ro_table[i] + i; + sum = counter + ro + pair[0] + pair[1]; + counter++; + __sync_fetch_and_add(&pair[1], 3); + return sum; +} + +typedef int (*op_fn)(int); + +static __noinline int add1(int v) +{ + return v + 1; +} + +/* + * Pointers to functions and to data in read-only data of a program with callx. + * Volatile, so that the compiler doesn't replace the pointers with what + * they point to. + */ +static const volatile struct { + op_fn fn; + int *data; + const char *name; +} ops SEC(".data.rel.ro") = { add1, &x, hello }; + +SEC("syscall") +int use_ops(void *ctx) +{ + /* a program has the arena when its code refers to it */ + counter++; +#ifdef __clang__ + return ops.fn(*ops.data) + ops.name[1]; +#else + /* gcc doesn't support indirect calls */ + return add1(*ops.data) + ops.name[1]; +#endif +} + +SEC("syscall") +int use_ptrs(void *ctx) +{ + unsigned long addr = (unsigned long)&aligned64; + char local[4] = "abc"; + + /* hide the address from the compiler, it knows that '& 63' is 0 */ + asm volatile ("" : "+r"(addr)); + if (addr & 63) + return 1; + aligned64.v[7] = 7; + if (*px != 42) + return 2; + *px = 43; + if (x != 43 || *cpx != 43) + return 3; + if (str[0] != 'h' || str[4] != 'o' || str[5]) + return 4; + /* the literal is in a map */ + if (bpf_strncmp(local, sizeof(local), "abc")) + return 5; + return 0; +} + +char out[16]; + +/* format strings are in a map */ +SEC("syscall") +int use_printk(void *ctx) +{ + char buf[sizeof(out)]; + int i, n; + + bpf_printk("counter %d", counter); + n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); + for (i = 0; i < sizeof(out); i++) + out[i] = buf[i]; + return n; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c new file mode 100644 index 0000000..01bc4fe --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c
@@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); +} arena SEC(".maps"); + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long sum; +const volatile int ro = 7; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +int __arena avar = 11; +#else +int avar = 11; +#endif + +SEC("syscall") +int use_data(void *ctx) +{ + sum = counter + ro + avar; + counter++; + avar++; + return sum; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c new file mode 100644 index 0000000..dc1ad5c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c
@@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +extern const int bpf_prog_active __ksym; + +/* the variable of the kernel is not in arena */ +const void *kp = &bpf_prog_active; + +SEC("syscall") +int ptr_to_extern(void *ctx) +{ + return *(int *)kp; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c new file mode 100644 index 0000000..ad8afe9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c
@@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int x = 42; +/* the table is read-only data with pointers. It's not in arena. */ +int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; +int *const volatile *pp = tbl; + +SEC("syscall") +int ptr_to_map(void *ctx) +{ + return **pp; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c new file mode 100644 index 0000000..ccf6632 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c
@@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +/* needs an arena map that is declared. The one that libbpf creates won't do. */ +int __arena_global avar = 11; + +SEC("syscall") +int arena_var(void *ctx) +{ + return avar + counter; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs new file mode 100644 index 0000000..3dd9d42 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs
@@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Why .data, .bss and .rodata of a program in Rust are in arena. +// +// A reference in Rust is an address. It doesn't say what it points to and it +// can be stored in data. The list below has a node in each of the sections. +// The nodes are linked by references that are in the data: +// IN_BSS.next is stored by the program, +// IN_DATA.next is a relocation in .data against .rodata. +// sum() loads the references back and reads the three nodes with the same insn. +// +// When the sections are array maps libbpf skips the relocation in .data, and +// what sum() loads from a node is a number that can't be dereferenced: +// R1 invalid mem access 'scalar' +// In arena the address of a node is a number to begin with. + +#![no_std] +#![no_main] + +// Tell libbpf to keep .data, .bss and .rodata in arena. +#[used] +#[link_section = ".arena.data"] +static DATA_IN_ARENA: u8 = 0; + +#[used] +#[link_section = "license"] +static LICENSE: [u8; 4] = *b"GPL\0"; + +// panic=abort is a stop gap until panic=unwind is supported. +// Nothing here panics, so the handler is not a part of the program. +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + loop {} +} + +pub struct Node { + val: u32, + next: Option<&'static Node>, +} + +// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. +#[no_mangle] +static IN_RODATA: Node = Node { val: 3, next: None }; +#[no_mangle] +static mut IN_DATA: Node = Node { + val: 20, + next: Some(&IN_RODATA), +}; +#[no_mangle] +static mut IN_BSS: Node = Node { val: 0, next: None }; + +#[inline(never)] +fn sum(mut node: Option<&Node>) -> u32 { + let mut sum = 0; + // The verifier wants a bound. + for _ in 0..8 { + let Some(n) = node else { break }; + sum += n.val; + node = n.next; + } + sum +} + +#[no_mangle] +#[link_section = "syscall"] +pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { + unsafe { + let head = &mut *&raw mut IN_BSS; + head.val = 100; + head.next = Some(&*&raw const IN_DATA); + sum(Some(head)) + } +}
diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 1cd61d7..c2247b3 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c
@@ -125,9 +125,9 @@ static int missing_release_callback_fn(__u32 index, void *data) return 0; } -/* Any dynptr initialized within a callback must have bpf_dynptr_put called */ +/* A callback cannot return with the last dynptr for a referenced resource. */ SEC("?raw_tp") -__failure __msg("Unreleased reference id") +__failure __msg("cannot overwrite referenced dynptr") int ringbuf_missing_release_callback(void *ctx) { bpf_loop(10, missing_release_callback_fn, NULL, 0); @@ -560,6 +560,9 @@ int global(void *ctx) /* A direct read should fail */ SEC("?raw_tp") __failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of dynptr stack state") +__msg("verifier-managed dynptr state") +__msg("Use dynptr helpers or kfuncs") int invalid_read1(void *ctx) { struct bpf_dynptr ptr; @@ -1892,6 +1895,101 @@ int clone_invalidate4(void *ctx) return 0; } +static __noinline void clone_slice_in_subprog(struct bpf_dynptr *ptr, int **data) +{ + struct bpf_dynptr clone; + + bpf_dynptr_clone(ptr, &clone); + *data = bpf_dynptr_data(&clone, 0, sizeof(val)); +} + +static __noinline void caller_slice_in_subprog(struct bpf_dynptr *ptr, int **data) +{ + struct bpf_dynptr clone; + + *data = bpf_dynptr_data(ptr, 0, sizeof(val)); + bpf_dynptr_clone(ptr, &clone); +} + +static __noinline void reserve_dynptr_in_subprog(void) +{ + struct bpf_dynptr ptr; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); +} + +/* A subprogram cannot lose the last dynptr that can release a resource. */ +SEC("?raw_tp") +__failure __msg("cannot overwrite referenced dynptr") +int referenced_dynptr_lost_on_subprog_return(void *ctx) +{ + reserve_dynptr_in_subprog(); + + return 0; +} + +/* + * Destroying a callee-local clone on return must not invalidate a slice whose + * source dynptr belongs to the caller. + */ +SEC("?raw_tp") +__success +int caller_dynptr_slice_across_subprog_valid(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + caller_slice_in_subprog(&ptr, &data); + if (data) + *data = 123; + bpf_ringbuf_submit_dynptr(&ptr, 0); + + return 0; +} + +/* + * A slice derived from a callee-local clone is invalid after the subprogram + * returns. + */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int callee_dynptr_slice_invalid_after_return(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + clone_slice_in_subprog(&ptr, &data); + if (data) + /* this should fail */ + *data = 123; + bpf_ringbuf_submit_dynptr(&ptr, 0); + + return 0; +} + +/* + * A slice from a caller-owned dynptr survives the subprogram return, but + * releasing the shared reservation must invalidate it. + */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int caller_dynptr_slice_release_after_subprog_invalid(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + caller_slice_in_subprog(&ptr, &data); + bpf_ringbuf_submit_dynptr(&ptr, 0); + if (data) + /* this should fail */ + *data = 123; + + return 0; +} + /* Invalidating a dynptr should invalidate any data slices * of its parent */ @@ -2050,19 +2148,18 @@ __noinline long global_call_bpf_dynptr(const struct bpf_dynptr *dynptr) /* Avoid leaving this global function empty to avoid having the compiler * optimize away the call to this global function. */ + __sink(dynptr); __sink(ret); return ret; } SEC("?raw_tp") -__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 type=trusted_ptr_ expected=fp, dynptr_ptr") int test_dynptr_reg_type(void *ctx) { - struct task_struct *current = NULL; - /* R1 should be holding a PTR_TO_BTF_ID, so this shouldn't be a - * reg->type that can be passed to a function accepting a - * ARG_PTR_TO_DYNPTR | MEM_RDONLY. process_dynptr_func() should catch - * this. + struct task_struct *current = bpf_get_current_task_btf(); + /* R1 holds a PTR_TO_BTF_ID, which cannot be passed to a function + * accepting ARG_PTR_TO_DYNPTR | MEM_RDONLY. */ global_call_bpf_dynptr((const struct bpf_dynptr *)current); return 0;
diff --git a/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c b/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c new file mode 100644 index 0000000..4f8b82a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c
@@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" + +/* + * Read-only and frozen, so the verifier sees a constant zero, while clang + * cannot fold the volatile read and keeps the call below. + */ +const volatile int never; + +/* Only called from dead code, hence removed as a whole before the JIT runs. */ +static __noinline int dead_subprog(struct __sk_buff *ctx) +{ + return ctx->len; +} + +__noinline int exception_cb_behind_dead_subprog_cb(u64 cookie) +{ + return 42; +} + +/* + * libbpf appends the exception callback behind the subprograms reached by + * calls, so removing the dead one compacts subprog_info[] underneath the + * callback's index. + */ +SEC("?tc") +__exception_cb(exception_cb_behind_dead_subprog_cb) +__success __retval(42) +int exception_cb_behind_dead_subprog(struct __sk_buff *ctx) +{ + if (never) + return dead_subprog(ctx); + bpf_throw(0); + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60..22503cf 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c
@@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, arena pointer, or a struct/union of scalars and arena pointers up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0);
diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c index aa2c05c..8b972fd 100644 --- a/tools/testing/selftests/bpf/progs/file_reader.c +++ b/tools/testing/selftests/bpf/progs/file_reader.c
@@ -27,9 +27,14 @@ char tmp_buf[256000]; int pid = 0; int err, run_success = 0; +__u64 beyond_eof_offset; static int validate_file_read(struct file *file); static int task_work_callback(struct bpf_map *map, void *key, void *value); +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value); + +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; SEC("lsm/file_open") int on_open_expect_fault(void *c) @@ -81,6 +86,101 @@ int on_open_validate_file_read(void *c) return 0; } +/* + * Exercise bpf_dynptr_from_file() first from a non-sleepable LSM program and + * then from its sleepable task-work callback. Reading beyond EOF makes the two + * backing implementations return different errors. + */ +SEC("lsm/file_open") +int on_open_non_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct elem *work; + struct file *file; + int key = 0; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 1; + return 0; + } + + /* The non-sleepable reader cannot fault in an uncached folio. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EFAULT) { + err = 2; + return 0; + } + + work = bpf_map_lookup_elem(&arrmap, &key); + if (!work) { + err = 3; + return 0; + } + + ret = bpf_task_work_schedule_signal(task, &work->tw, &arrmap, + sleepable_second_callback); + if (ret) + err = 4; + return 0; +} + +/* + * Exercise the opposite fixup order: the first call is made from a sleepable + * LSM program, while the RCU read-side section makes the second non-sleepable. + */ +SEC("lsm.s/file_open") +int on_open_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 7; + return 0; + } + + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EIO) { + err = 8; + goto out; + } + + bpf_rcu_read_lock(); + ret = bpf_dynptr_from_file(file, 0, &dynptr); + bpf_rcu_read_unlock(); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EFAULT) { + err = 9; + goto out; + } + + run_success = 1; +out: + bpf_put_file(file); + return 0; +} + /* Called in a sleepable context, read 256K bytes, cross check with user space read data */ static int task_work_callback(struct bpf_map *map, void *key, void *value) { @@ -97,6 +197,35 @@ static int task_work_callback(struct bpf_map *map, void *key, void *value) return 0; } +/* Task-work callbacks are verified as sleepable. */ +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 5; + return 0; + } + + /* freader_fetch() converts __kernel_read()'s short read at EOF to -EIO. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EIO) { + err = 6; + return 0; + } + + run_success = 1; + return 0; +} + static int verify_dynptr_read(struct bpf_dynptr *ptr, u32 off, char *user_buf, u32 len) { int i;
diff --git a/tools/testing/selftests/bpf/progs/freplace_ret_pair.c b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c new file mode 100644 index 0000000..12c15d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c
@@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +SEC("freplace/agg_ret_target_func") +__u64 new_agg_ret_target_func(void) +{ + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/get_func_args_test.c b/tools/testing/selftests/bpf/progs/get_func_args_test.c index 1bf47f6..9b0beb5 100644 --- a/tools/testing/selftests/bpf/progs/get_func_args_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_args_test.c
@@ -6,6 +6,40 @@ char _license[] SEC("license") = "GPL"; +/* Keep the upper index bits: the inlined expansion must truncate them. */ +static long (*bpf_get_func_arg_wide)(void *ctx, __u64 n, __u64 *value) = + (void *)BPF_FUNC_get_func_arg; + +/* Both traced functions are called with arguments 2 and 3. */ +static __always_inline int check_wide_index(void *ctx) +{ + __u64 a = 0, z = 0xdeadbeefdeadbeefULL; + __s64 err; + int ok = 1; + + /* The index is truncated to u32, so these select arguments 0 and 1. */ + err = bpf_get_func_arg_wide(ctx, 1ULL << 32, &a); + ok &= err == 0 && ((int)a == 2); + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 1, &a); + ok &= err == 0 && a == 3; + err = bpf_get_func_arg_wide(ctx, 1ULL << 59, &a); + ok &= err == 0 && ((int)a == 2); + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 1, &a); + ok &= err == 0 && a == 3; + + /* Out of range: -EINVAL and no store into the destination. */ + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 2, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 0xffffffffULL, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 2, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 0xffffffffULL, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + + return ok; +} + __u64 test1_result = 0; SEC("fentry/bpf_fentry_test1") int BPF_PROG(test1) @@ -63,6 +97,7 @@ int BPF_PROG(test2) /* return value */ err = bpf_get_func_ret(ctx, &ret); test2_result &= err == 0 && ret == 5; + test2_result &= check_wide_index(ctx); return 0; } @@ -162,6 +197,7 @@ int BPF_PROG(tp_test2) /* not valid argument */ err = bpf_get_func_arg(ctx, 2, &z); test6_result &= err == -EINVAL; + test6_result &= check_wide_index(ctx); return 0; }
diff --git a/tools/testing/selftests/bpf/progs/irq.c b/tools/testing/selftests/bpf/progs/irq.c index a4a0078..50727fa 100644 --- a/tools/testing/selftests/bpf/progs/irq.c +++ b/tools/testing/selftests/bpf/progs/irq.c
@@ -15,7 +15,20 @@ struct bpf_res_spin_lock lockA __hidden SEC(".data.A"); struct bpf_res_spin_lock lockB __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("R1 doesn't point to an irq flag on stack") +__failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of IRQ flag stack state") +__msg("verifier-managed IRQ flag state") +__msg("Pass the saved IRQ flag to the matching restore kfunc") +int irq_flag_direct_read(struct __sk_buff *ctx) +{ + unsigned long flags; + + bpf_local_irq_save(&flags); + return flags; +} + +SEC("?tc") +__failure __msg("R1 type=map_value expected=fp") int irq_save_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_save(&global_flags); @@ -23,7 +36,7 @@ int irq_save_bad_arg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 doesn't point to an irq flag on stack") +__failure __msg("R1 type=map_value expected=fp") int irq_restore_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_restore(&global_flags);
diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index c669915..ae845ad 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c
@@ -1688,7 +1688,7 @@ int iter_subprog_check_stacksafe(const void *ctx) struct bpf_iter_num global_it; SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_new_bad_arg(const void *ctx) { bpf_iter_num_new(&global_it, 0, 1); @@ -1696,7 +1696,7 @@ int iter_new_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_next_bad_arg(const void *ctx) { bpf_iter_num_next(&global_it); @@ -1704,7 +1704,7 @@ int iter_next_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_destroy_bad_arg(const void *ctx) { bpf_iter_num_destroy(&global_it); @@ -2188,4 +2188,386 @@ __naked void loop_counter_precision_2nd_iter(void) ); } +/* + * An iterator that is destroyed and created again inside its loop starts from + * the beginning and the loop never ends. In the first four progs the first + * prune point after bpf_iter_num_new() is the call of bpf_iter_num_next(). + */ +SEC("socket") +__description("iter: remake in the loop, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 != 0 goto 3f;" + "r1 = r10;" + "r1 += -8;" +"2:" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" +"3:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 2b;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +__used __naked static void iter_remake(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_iter_num_destroy];" + "r1 = r6;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_destroy) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake in a callee, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_callee_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 != 0 goto 3f;" + "r1 = r10;" + "r1 += -8;" +"2:" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" +"3:" + "r1 = r10;" + "r1 += -8;" + "call iter_remake;" + "r1 = r10;" + "r1 += -8;" + "goto 2b;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake after next, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_after_next_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" +"1:" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: two iterators, remake each after its next") +__failure __msg("infinite loop detected") +__naked void iter_remake_two_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -16;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" +"1:" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -16;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* Same loops with a prune point in front of bpf_iter_num_next() */ +SEC("socket") +__description("iter: remake in the loop") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_inline(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake in a callee") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_callee(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call iter_remake;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake after next") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_after_next(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake when drained") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_drained(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 119 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* The inner iterator is created in every iteration of the outer loop */ +SEC("socket") +__description("iter: nested loops") +__success __retval(12) +int iter_nested_remake(void *ctx) +{ + int i, j, sum = 0; + + bpf_for(i, 0, 3) { + bpf_for(j, 0, 4) + sum++; + } + return sum; +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/iters_state_safety.c b/tools/testing/selftests/bpf/progs/iters_state_safety.c index e5bb9fe..b2ba460 100644 --- a/tools/testing/selftests/bpf/progs/iters_state_safety.c +++ b/tools/testing/selftests/bpf/progs/iters_state_safety.c
@@ -354,6 +354,9 @@ int next_after_destroy_fail(void *ctx) SEC("?raw_tp") __failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of iterator stack state") +__msg("verifier-managed iterator state") +__msg("Use iterator kfuncs") int __naked read_from_iter_slot_fail(void) { asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 76012db..145750bb 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c
@@ -46,6 +46,17 @@ int iter_next_trusted_or_null(const void *ctx) return 0; } +SEC("syscall") +__success +int trusted_nullable_kfunc(const void *ctx) +{ + struct task_struct *task; + + task = bpf_kfunc_trusted_null_test(); + bpf_kfunc_trusted_nullable_test(task); + return 0; +} + SEC("raw_tp/sys_enter") __success int iter_next_rcu(const void *ctx) @@ -105,8 +116,7 @@ int iter_next_rcu_not_trusted(const void *ctx) } SEC("raw_tp/sys_enter") -__failure __msg("R1 cannot write into rdonly_mem") -/* Message should not be 'R1 cannot write into rdonly_trusted_mem' */ +__failure __msg("R1 type=rdonly_mem expected=fp") int iter_next_ptr_mem_not_trusted(const void *ctx) { struct bpf_iter_num num_it; @@ -135,7 +145,7 @@ int iter_ret_rcu_test_protected(const void *ctx) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -__failure __msg("R1 type=rcu_ptr_or_null_ expected=") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_ret_rcu_test_type(const void *ctx) { struct task_struct *p; @@ -158,7 +168,7 @@ int iter_ret_rcu_test_protected_nostruct(const void *ctx) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -__failure __msg("R1 type=rdonly_rcu_mem_or_null expected=") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_ret_rcu_test_type_nostruct(const void *ctx) { void *p;
diff --git a/tools/testing/selftests/bpf/progs/jit_insns_arena.c b/tools/testing/selftests/bpf/progs/jit_insns_arena.c new file mode 100644 index 0000000..95736ea --- /dev/null +++ b/tools/testing/selftests/bpf/progs/jit_insns_arena.c
@@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The arena accesses of jit_insns in their own object, so that a kernel + * without arena support fails only these programs. + */ +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The first bytes in memory of the test value and its sum with the slot + * after the u32 store depend on the byte order. + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define VAL_B0 0x88 +#define VAL_H0 0x7788 +#define VAL_W0 0x55667788 +#define VAL_SUM 0x22446688556689bc +#else +#define VAL_B0 0x11 +#define VAL_H0 0x1122 +#define VAL_W0 0x11223344 +#define VAL_SUM 0x11224578aaccef10 +#endif + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +SEC("syscall") +__success __retval(0) +int arena_zero_off(void *ctx) +{ + void __arena *page; + __u64 res; + + page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page) + return 1; + res = (__u32)(__u64)page; + /* Accesses through two base registers */ + asm volatile (" \ + r7 = %[res]; \ + r7 = addr_space_cast(r7, 0x0, 0x1); \ + r1 = r7; \ + r8 = 0x1122334455667788 ll; \ + *(u64 *)(r7 + 0) = r8; \ + r9 = *(u64 *)(r7 + 0); \ + r9 ^= r8; \ + *(u64 *)(r1 + 0) = r8; \ + r2 = *(u64 *)(r1 + 0); \ + r2 ^= r8; \ + r9 |= r2; \ + r3 = *(u8 *)(r1 + 0); \ + r3 ^= %[b0]; \ + r9 |= r3; \ + r4 = *(u16 *)(r1 + 0); \ + r4 ^= %[h0]; \ + r9 |= r4; \ + r5 = *(u32 *)(r1 + 0); \ + r5 ^= %[w0]; \ + r9 |= r5; \ + *(u32 *)(r1 + 0) = 0x1234; \ + lock *(u64 *)(r7 + 0) += r8; \ + r2 = *(u64 *)(r1 + 0); \ + r8 = %[sum] ll; \ + r2 ^= r8; \ + r9 |= r2; \ + %[res] = r9; \ +" : [res]"+r"(res) + : __imm_const(b0, VAL_B0), __imm_const(h0, VAL_H0), + __imm_const(w0, VAL_W0), __imm_const(sum, VAL_SUM) + : "r1", "r2", "r3", "r4", "r5", "r7", "r8", "r9"); + return res != 0; +} + +SEC("syscall") +__success __retval(0) +int cast_to_user(void *ctx) +{ + void __arena *page; + __u64 res; + + page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page) + return 1; + /* The user address comes back from the kernel one, NULL stays NULL */ + asm volatile (" \ + r7 = %[page]; \ + r1 = addr_space_cast(r7, 0x0, 0x1); \ + r1 = addr_space_cast(r1, 0x1, 0x0); \ + r9 = r1; \ + r9 ^= r7; \ + r2 = 0; \ + r2 = addr_space_cast(r2, 0x1, 0x0); \ + r9 |= r2; \ + %[res] = r9; \ +" : [res]"=r"(res) + : [page]"r"(page) + : "r1", "r2", "r7", "r9"); + return res != 0; +} +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/kasan.c b/tools/testing/selftests/bpf/progs/kasan.c new file mode 100644 index 0000000..fe8e0dd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan.c
@@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + +extern struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) __ksym; +extern void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) __ksym; + +int access_size; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val.data_1 = 0xAA; + break; + case 2: + val.data_2 = 0xAA; + break; + case 4: + val.data_4 = 0xAA; + break; + case 8: + val.data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int st_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = 0xAA; + break; + case 2: + val->data_2 = 0xAA; + break; + case 4: + val->data_4 = 0xAA; + break; + case 8: + val->data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int stx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + /* + * Unlike the st() programs above, the stored value comes from a + * runtime source (skb->len), so it cannot be constant-folded and + * clang always emits a genuine BPF_STX (register store) regardless + * of the target cpu version. + */ + switch (access_size) { + case 1: + val.data_1 = (__u8)skb->len; + break; + case 2: + val.data_2 = (__u16)skb->len; + break; + case 4: + val.data_4 = (__u32)skb->len; + break; + case 8: + val.data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int stx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = (__u8)skb->len; + break; + case 2: + val->data_2 = (__u16)skb->len; + break; + case 4: + val->data_4 = (__u32)skb->len; + break; + case 8: + val->data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val.data_1); + break; + case 2: + __sink(val.data_2); + break; + case 4: + __sink(val.data_4); + break; + case 8: + __sink(val.data_8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val->data_1); + break; + case 2: + __sink(val->data_2); + break; + case 4: + __sink(val->data_4); + break; + case 8: + __sink(val->data_8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sink(val->data_4); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sink(val.data_4); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val.data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val.data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val->data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val->data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val.data_8, 8); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val->data_8, 8); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +#ifdef __BPF_FEATURE_LOAD_ACQ_STORE_REL +bool skip_load_acq_store_rel_tests SEC(".data") = 0; + +SEC("tcx/ingress") +int load_acquire_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val.data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val.data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val.data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val.data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int load_acquire_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val->data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val->data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val->data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val->data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val.data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val.data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val.data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val.data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val->data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val->data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val->data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val->data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} +#else +bool skip_load_acq_store_rel_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int verifier_paths_stack_and_non_stack(struct __sk_buff *skb) +{ + struct kasan_test_val stack_val = {}; + struct kasan_test_val *val; + void *ptr; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + if (access_size) + ptr = val; + else + ptr = &stack_val; + + bpf_kfunc_kasan_poison(val, sizeof(*val)); + *(__u8 *)ptr = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(*val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_oob(struct __sk_buff *skb) +{ + struct bpf_testmod_oob *val; + struct kasan_test_val volatile tmp; + + val = bpf_testmod_oob_alloc(); + if (!val) + return 0; + + switch (access_size) { + case 1: + tmp.data_1 = (__u8)val->redzone_1; + break; + case 2: + tmp.data_2 = (__u16)val->redzone_2; + break; + case 4: + tmp.data_4 = (__u32)val->redzone_4; + break; + case 8: + tmp.data_8 = (__u64)val->redzone_8; + break; + } + bpf_testmod_oob_free(val); + return tmp.data_1; +} + +SEC("tcx/ingress") +int ldx_self_alias_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + __u64 addr; + + bpf_kfunc_kasan_poison(&val, sizeof(val)); + /* + * Check that a stack access with dst_reg == src_reg is correctly + * flagged as stack-only access + */ + addr = (__u64)&val; + asm volatile( + "r1 = %0\n" + "r1 = *(u64 *)(r1 + 0)\n" + : + : "r"(addr) + : "r1", "memory"); + bpf_kfunc_kasan_unpoison(&val, sizeof(val)); + + return 0; +} + +char LICENSE[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/kasan_harden.c b/tools/testing/selftests/bpf/progs/kasan_harden.c new file mode 100644 index 0000000..8c9eb20 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan_harden.c
@@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_blinded(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + val->data_1 = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +char LICENSE[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/ksock_common.h b/tools/testing/selftests/bpf/progs/ksock_common.h index 01edaeb..2a32cff 100644 --- a/tools/testing/selftests/bpf/progs/ksock_common.h +++ b/tools/testing/selftests/bpf/progs/ksock_common.h
@@ -16,6 +16,8 @@ int bpf_ksock_connect(struct bpf_ksock *ks, const union bpf_ksock_addr *addr, struct bpf_ksock *bpf_ksock_acquire(struct bpf_ksock *ks) __ksym; void bpf_ksock_release(struct bpf_ksock *ks) __ksym; int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) __ksym; +int bpf_ksock_sendto(struct bpf_ksock *ks, const void *data, u32 data__sz, + const union bpf_ksock_addr *addr__nullable, u32 addr__sz) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym;
diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c index fd2ccfd..9533570 100644 --- a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c +++ b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c
@@ -12,7 +12,7 @@ char send_data[11] = "dummy data"; SEC("lsm.s/socket_sendmsg") __description("bpf_ksock_send is rejected from socket_sendmsg LSM hook") __failure __msg("calling kernel function bpf_ksock_send is not allowed") -int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, +int BPF_PROG(socket_sendmsg_ksock_send, struct socket *sock, struct msghdr *msg, int size, int ret) { struct __ksock_ctx_value *v; @@ -32,4 +32,27 @@ int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, return ret; } +SEC("lsm.s/socket_sendmsg") +__description("bpf_ksock_sendto is rejected from socket_sendmsg LSM hook") +__failure __msg("calling kernel function bpf_ksock_sendto is not allowed") +int BPF_PROG(socket_sendmsg_ksock_sendto, struct socket *sock, + struct msghdr *msg, int size, int ret) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks; + + v = ksock_ctx_value_lookup(); + if (!v) + return ret; + + ks = bpf_kptr_xchg(&v->ctx, NULL); + if (!ks) + return ret; + + bpf_ksock_sendto(ks, send_data, sizeof(send_data), NULL, 0); + bpf_ksock_release(ks); + + return ret; +} + char __license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/ksock_sendto.c b/tools/testing/selftests/bpf/progs/ksock_sendto.c new file mode 100644 index 0000000..b6eacf2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_sendto.c
@@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> +#include "bpf_tracing_net.h" +#include "ksock_common.h" + +char sendto_data[] = "hello from bpf sendto"; +char send_data[] = "hello from bpf send"; + +__be16 connected_port; +__be16 sendto_port; +bool connected; +bool use_sendto_addr; +int sendto_ret = -1; +int send_ret = -1; + +SEC("syscall") +int test_sendto(void *ctx) +{ + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + union bpf_ksock_addr addr = {}; + struct bpf_ksock *ks; + int err = 0; + + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (!ks) + return err; + + addr.sin.sin_family = AF_INET; + addr.sin.sin_addr.s_addr = bpf_htonl(0x7f000001); + if (connected) { + addr.sin.sin_port = connected_port; + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (err) + goto out; + } + + addr.sin.sin_port = sendto_port; + if (use_sendto_addr) + sendto_ret = bpf_ksock_sendto(ks, sendto_data, sizeof(sendto_data), + &addr, sizeof(addr)); + else + sendto_ret = bpf_ksock_sendto(ks, sendto_data, sizeof(sendto_data), NULL, 0); + + /* sendto() must neither connect the socket nor change its default peer. */ + send_ret = bpf_ksock_send(ks, send_data, sizeof(send_data)); +out: + bpf_ksock_release(ks); + return err; +} + +char __license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/ksock_wq.c b/tools/testing/selftests/bpf/progs/ksock_wq.c index 16a1873..fab3562 100644 --- a/tools/testing/selftests/bpf/progs/ksock_wq.c +++ b/tools/testing/selftests/bpf/progs/ksock_wq.c
@@ -2,6 +2,7 @@ /* Copyright (c) 2026 Isovalent */ #include "vmlinux.h" +#include <bpf/bpf_endian.h> #include <bpf/bpf_helpers.h> #include "bpf_experimental.h" #include "bpf_tracing_net.h" @@ -14,15 +15,19 @@ struct ksock_wq_value { struct { __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, 1); + __uint(max_entries, 2); __type(key, u32); __type(value, struct ksock_wq_value); } work_map SEC(".maps"); -int create_err; +char send_data[] = "hello from bpf workqueue"; + +__be16 remote_port; +int connect_send_ret; +int create_ret; u32 callback_done; -static int ksock_wq_callback(void *map, int *key, void *value) +static int ksock_wq_create_cb(void *map, int *key, void *value) { struct bpf_ksock_create_opts opts = { .family = AF_INET, @@ -35,28 +40,85 @@ static int ksock_wq_callback(void *map, int *key, void *value) ks = bpf_ksock_create(&opts, sizeof(opts), &err); if (ks) bpf_ksock_release(ks); - create_err = err; + create_ret = err; __sync_fetch_and_add(&callback_done, 1); return 0; } +static int ksock_wq_connect_send_cb(void *map, int *key, void *value) +{ + union bpf_ksock_addr addr = { + .sin = { + .sin_family = AF_INET, + .sin_port = remote_port, + .sin_addr.s_addr = bpf_htonl(0x7f000001), + }, + }; + struct bpf_ksock *ks; + int err = -ENOENT; + + ks = ksock_ctx_get(); + if (!ks) + goto out; + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (!err) + err = bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); +out: + connect_send_ret = err; + __sync_fetch_and_add(&callback_done, 1); + return 0; +} + +static int wq_set_callback_start(int (*callback_fn)(void *, int *, void *), u32 key) +{ + struct ksock_wq_value *value; + int err; + + value = bpf_map_lookup_elem(&work_map, &key); + if (!value) + return -ENOENT; + + err = bpf_wq_init(&value->work, &work_map, 0); + if (err) + return err; + err = bpf_wq_set_callback(&value->work, callback_fn, 0); + if (err) + return err; + err = bpf_wq_start(&value->work, 0); + if (err) + return err; + + return 0; +} + SEC("syscall") int ksock_wq_start(void *ctx) { - struct ksock_wq_value *value; - u32 key = 0; - int err; + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + struct bpf_ksock *ks; + int err = 0; - value = bpf_map_lookup_elem(&work_map, &key); - if (!value) - return -ENOENT; - err = bpf_wq_init(&value->work, &work_map, 0); + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (!ks) + return err; + err = ksock_ctx_insert(ks); if (err) return err; - err = bpf_wq_set_callback(&value->work, ksock_wq_callback, 0); + + err = wq_set_callback_start(ksock_wq_connect_send_cb, 0); if (err) return err; - return bpf_wq_start(&value->work, 0); + + err = wq_set_callback_start(ksock_wq_create_cb, 1); + if (err) + return err; + + return 0; } char __license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/linked_arena1.c b/tools/testing/selftests/bpf/progs/linked_arena1.c new file mode 100644 index 0000000..5027b4a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/linked_arena1.c
@@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); /* number of pages */ +} arena SEC(".maps"); + +/* + * Dereferencing an arena global needs the compiler to emit an + * addr_space_cast, which only clang does. Keep the variables so the arena + * is still populated and the skeleton still has its arena member, but skip + * the test elsewhere. + */ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST +bool skip_tests __attribute((__section__(".data"))) = false; +#else +bool skip_tests = true; +#endif + +long __arena_global a_val = 1; +extern long __arena_global b_val; /* defined in linked_arena2.c */ + +SEC("syscall") +int sum1(void *ctx) +{ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + return a_val + b_val; +#else + /* + * Reference the extern without dereferencing it, so that the relink + * test still has an extern to resolve. + */ + return (long)&b_val; +#endif +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/linked_arena2.c b/tools/testing/selftests/bpf/progs/linked_arena2.c new file mode 100644 index 0000000..db37198 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/linked_arena2.c
@@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +long __arena_global b_val = 2; +extern long __arena_global a_val; /* defined in linked_arena1.c */ + +SEC("syscall") +int bump2(void *ctx) +{ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + a_val += 10; + b_val += 20; + return a_val + b_val; +#else /* see linked_arena1.c */ + return (long)&a_val; +#endif +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/lirc_mode2.c b/tools/testing/selftests/bpf/progs/lirc_mode2.c new file mode 100644 index 0000000..98137f3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lirc_mode2.c
@@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young <sean@mess.org> + +#include <linux/bpf.h> +#include <linux/lirc.h> +#include <bpf/bpf_helpers.h> + +SEC("lirc_mode2") +int bpf_decoder(unsigned int *sample) +{ + if (LIRC_IS_PULSE(*sample)) { + unsigned int duration = LIRC_VALUE(*sample); + + /* + * Flag bits picked deliberately low: rc-loopback simulates + * a receiver overflow for any pulse over MS_TO_US(50) (see + * loop_tx_ir() in rc-loopback.c), which would silently + * swallow the sample before it ever reaches this decoder. + */ + if (duration & 0x8000) + bpf_rc_keydown(sample, 0x40, duration & 0x3fff, 0); + if (duration & 0x4000) + bpf_rc_pointer_rel(sample, (duration >> 7) & 0x7f, + duration & 0x7f); + } + + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/lsm.c b/tools/testing/selftests/bpf/progs/lsm.c index 7e32fbd..e210e07 100644 --- a/tools/testing/selftests/bpf/progs/lsm.c +++ b/tools/testing/selftests/bpf/progs/lsm.c
@@ -192,6 +192,7 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) { __u32 pid = bpf_get_current_pid_tgid() >> 32; struct inner_map *inner_map; + struct mm_struct *mm; char args[64]; __u32 key = 0; __u64 *value; @@ -200,7 +201,9 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) bprm_count++; bpf_copy_from_user(args, sizeof(args), (void *)bprm->vma->vm_mm->arg_start); - bpf_copy_from_user(args, sizeof(args), (void *)bprm->mm->arg_start); + mm = bprm->mm; + if (mm) + bpf_copy_from_user(args, sizeof(args), (void *)mm->arg_start); value = bpf_map_lookup_elem(&array, &key); if (value)
diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c new file mode 100644 index 0000000..77b37cf --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c
@@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#define LABEL_MAX 32 + +struct label { + char v[LABEL_MAX]; + __u32 len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_INODE_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct label); +} inode_zone SEC(".maps"); + +__u32 monitored_pid; + +const char xattr_zone[] = "security.bpf.zone"; +const char xattr_origin[] = "security.bpf.origin"; +const char xattr_overflow[] = "security.bpf.overflow"; +/* One byte over XATTR_NAME_MAX once "security." is prepended again. */ +const char xattr_toolong[] = "security.bpf." "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const char xattr_selinux[] = "security.selinux"; +const char xattr_user[] = "user.zone"; + +char zone_scratch[LABEL_MAX]; +char origin_value[] = "created"; + +__u32 hook_ran; +__s32 zone_err = 1; +__s32 origin_err = 1; +__s32 overflow_err = 1; +__s32 toolong_err = 1; +__s32 selinux_err = 1; +__s32 user_err = 1; +__u32 inherited; + +SEC("lsm/inode_init_security") +int BPF_PROG(init_label, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + struct label *parent; + int len = 0; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + + hook_ran = 1; + if (!xattrs) { + /* The filesystem takes no xattrs at inode creation. */ + zone_err = -EOPNOTSUPP; + return 0; + } + __builtin_memset(zone_scratch, 0, LABEL_MAX); + + if (dir) { + parent = bpf_inode_storage_get(&inode_zone, dir, 0, 0); + if (parent && parent->len > 0 && parent->len <= LABEL_MAX) { + len = parent->len; + __builtin_memcpy(zone_scratch, parent->v, LABEL_MAX); + inherited = 1; + } + } + if (!len) { + __builtin_memcpy(zone_scratch, "default", sizeof("default")); + len = sizeof("default"); + } + + bpf_dynptr_from_mem(zone_scratch, len, 0, &value); + /* Refused before a slot is claimed, so the budget below is intact. */ + toolong_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_toolong, + &value); + zone_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, + &value); + + bpf_dynptr_from_mem(origin_value, sizeof(origin_value), 0, &value); + origin_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_origin, + &value); + + overflow_err = bpf_inode_init_xattr(xattrs, xattr_count, + xattr_overflow, &value); + + selinux_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_selinux, + &value); + user_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_user, + &value); + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c new file mode 100644 index 0000000..08dd93a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c
@@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +const char xattr_name[32] = "security.bpf.slot"; + +__u32 monitored_pid; +__u32 hook_ran; +__s32 claim_err = 1; + +char claim_value[] = "v"; + +SEC("lsm/inode_init_security") +int BPF_PROG(claim_one, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + + hook_ran = 1; + if (!xattrs) { + claim_err = -EOPNOTSUPP; + return 0; + } + bpf_dynptr_from_mem(claim_value, sizeof(claim_value), 0, &value); + claim_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_name, + &value); + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c new file mode 100644 index 0000000..6a879a7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c
@@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#define VALUE_MAX (64 * 1024 + 1) + +const char xattr_value[] = "security.bpf.value"; + +/* Shape of the value handed to the kfunc, and what the hook returns after. */ +const volatile __u32 value_len; +const volatile __u64 dynptr_flags; +const volatile __s32 hook_retval; + +char value_src[VALUE_MAX]; + +__u32 monitored_pid; +__u32 hook_ran; +__s32 claim_err = 1; + +SEC("lsm/inode_init_security") +int BPF_PROG(claim_value, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + __u32 len = value_len; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + if (len > VALUE_MAX) + return 0; + + hook_ran = 1; + if (!xattrs) { + claim_err = -EOPNOTSUPP; + return 0; + } + bpf_dynptr_from_mem(value_src, len, dynptr_flags, &value); + claim_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_value, + &value); + return hook_retval; +}
diff --git a/tools/testing/selftests/bpf/progs/map_excl.c b/tools/testing/selftests/bpf/progs/map_excl.c index d461684..90f1270 100644 --- a/tools/testing/selftests/bpf/progs/map_excl.c +++ b/tools/testing/selftests/bpf/progs/map_excl.c
@@ -27,7 +27,8 @@ int should_have_access(void *ctx) SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") int should_not_have_access(void *ctx) { - int key = 0, value = 0xdeadbeef; + /* a different value, so that the two programs do not share a digest */ + int key = 0, value = 0xbaadf00d; bpf_map_update_elem(&excl_map, &key, &value, 0); return 0;
diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index eee35d2..ac4003b 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c
@@ -149,7 +149,7 @@ int reject_bad_type_match(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 type=untrusted_ptr_or_null_ expected=percpu_ptr_") +__failure __msg("Possibly NULL pointer passed to trusted R1") int marked_as_untrusted_or_null(struct __sk_buff *ctx) { struct map_value *v; @@ -217,7 +217,7 @@ int reject_kptr_xchg_on_unref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 type=rcu_ptr_or_null_ expected=percpu_ptr_") +__failure __msg("Possibly NULL pointer passed to trusted R1") int mark_ref_as_untrusted_or_null(struct __sk_buff *ctx) { struct map_value *v; @@ -252,7 +252,7 @@ int reject_untrusted_store_to_ref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("release helper bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") +__failure __msg("release function bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int reject_untrusted_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; @@ -291,7 +291,7 @@ int reject_bad_type_xchg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("invalid kptr access, R2 type=trusted_ptr_prog_test_ref_kfunc") +__failure __msg("R2 must have zero offset when passed to release func") int reject_member_of_ref_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *ref_ptr; @@ -364,7 +364,7 @@ int kptr_xchg_ref_state(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to helper R2") +__success int kptr_xchg_possibly_null(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p;
diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index 373c8d1..6bd4cb6 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c
@@ -114,7 +114,7 @@ static inline int check_hash(void) VERIFY(check_default_noinline(&hash->map, map)); VERIFY(hash->n_buckets == MAX_ENTRIES); - VERIFY(hash->elem_size == 64); + VERIFY(hash->elem_size == 32); VERIFY(hash->count.counter == 0); VERIFY(bpf_map_sum_elem_count(map) == 0);
diff --git a/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c new file mode 100644 index 0000000..5c2aa0a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c
@@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* r0-r5 after the loop and the number of iterations */ +__u64 regs_64[7]; +__u64 regs_128[7]; + +/* + * fentry prog with 64 or more bytes of stack uses private stack. + * Loop until may_goto expires. r0-r5 should stay zero. + */ +#define TIMED_MAY_GOTO_PRIV_STACK(size, func) \ +SEC("fentry/" #func) \ +__naked void priv_stack_##size(void) \ +{ \ + asm volatile ( \ + "r6 = 0;" \ + "*(u64 *)(r10 - " #size ") = r6;" \ + "r7 = %[regs] ll;" \ + "r0 = 0;" \ + "r1 = 0;" \ + "r2 = 0;" \ + "r3 = 0;" \ + "r4 = 0;" \ + "r5 = 0;" \ +"1:" \ + ".8byte %[may_goto];" \ + "r6 += 1;" \ + "goto 1b;" \ + "*(u64 *)(r7 + 0) = r0;" \ + "*(u64 *)(r7 + 8) = r1;" \ + "*(u64 *)(r7 + 16) = r2;" \ + "*(u64 *)(r7 + 24) = r3;" \ + "*(u64 *)(r7 + 32) = r4;" \ + "*(u64 *)(r7 + 40) = r5;" \ + "*(u64 *)(r7 + 48) = r6;" \ + "r0 = 0;" \ + "exit;" \ + : \ + : [regs]"i"(®s_##size), \ + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) \ + : __clobber_all); \ +} + +TIMED_MAY_GOTO_PRIV_STACK(64, bpf_fentry_test1) +TIMED_MAY_GOTO_PRIV_STACK(128, bpf_fentry_test2) + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index 3e0d4f6..2301902 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c
@@ -118,8 +118,7 @@ int atomic_rmw_not_ok(void *ctx) SEC("socket") __failure -__msg("invalid access to memory, mem_size=0 off=0 size=4") -__msg("R1 min value is outside of the allowed memory range") +__msg("R1 type=rdonly_untrusted_mem expected=fp") int kfunc_param_not_ok(void *ctx) { int *p;
diff --git a/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c new file mode 100644 index 0000000..b551192 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c
@@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +struct reclaim_args { + __u64 cgroup_id; + __u64 size; +}; + +/* Signed, because bpf_test_run_opts.retval is a __u32. */ +__s64 reclaimed; + +SEC("syscall") +int memcg_proactive_reclaim(struct reclaim_args *ctx) +{ + struct mem_cgroup *memcg; + struct cgroup *cgrp; + + cgrp = bpf_cgroup_from_id(ctx->cgroup_id); + if (!cgrp) + return 0; + + memcg = bpf_get_mem_cgroup(&cgrp->self); + if (memcg) { + reclaimed = bpf_proactive_reclaim(memcg, ctx->size, -1); + bpf_put_mem_cgroup(memcg); + } + bpf_cgroup_release(cgrp); + + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c index 3701f4e..57615b0 100644 --- a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c +++ b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c
@@ -24,6 +24,7 @@ struct val_600b_t { struct elem { long sum; struct val_t __percpu_kptr *pc; + struct val_t __percpu_kptr *pc2; }; struct { @@ -46,6 +47,8 @@ struct { struct task_struct *bpf_task_from_pid(s32 pid) __ksym; void bpf_task_release(struct task_struct *p) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; long ret; @@ -124,6 +127,38 @@ int BPF_PROG(test_array_map_3) } SEC("?fentry.s/bpf_fentry_test1") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(reject_nullable_percpu_xchg_alias) +{ + struct val_t __percpu_kptr *p1, *p2, *old; + struct val_t *v; + struct elem *e; + int index = 0; + + e = bpf_map_lookup_elem(&array, &index); + if (!e) + return 0; + + p1 = bpf_percpu_obj_new(struct val_t); + p2 = bpf_percpu_obj_new(struct val_t); + + bpf_rcu_read_lock(); + old = bpf_kptr_xchg(&e->pc, p1); + if (old) + bpf_percpu_obj_drop(old); + old = bpf_kptr_xchg(&e->pc2, p2); + if (old) + bpf_percpu_obj_drop(old); + + if (p1) { + v = bpf_this_cpu_ptr(p2); + v->b = 1; + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("?fentry.s/bpf_fentry_test1") __failure __msg("R1 expected for bpf_percpu_obj_drop()") int BPF_PROG(test_array_map_4) {
diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c index 81c4594..955391d 100644 --- a/tools/testing/selftests/bpf/progs/preempt_lock.c +++ b/tools/testing/selftests/bpf/progs/preempt_lock.c
@@ -6,6 +6,8 @@ #include "bpf_experimental.h" extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; +extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; SEC("?tc") __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") @@ -179,6 +181,30 @@ int preempt_sleepable_kfunc(void *ctx) return 0; } +SEC("?fentry/" SYS_PREFIX "sys_getpgid") +__failure __msg("program must be sleepable to call sleepable kfunc bpf_copy_from_user_str") +__msg("cannot be used in non-sleepable program") +int non_sleepable_kfunc(void *ctx) +{ + u32 data; + + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("kernel func bpf_copy_from_user_str is sleepable within rcu_read_lock region") +__msg("cannot be used in RCU read lock region") +int sleepable_kfunc_in_rcu(void *ctx) +{ + u32 data; + + bpf_rcu_read_lock(); + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + bpf_rcu_read_unlock(); + return 0; +} + int __noinline preempt_global_subprog(void) { preempt_balance_subprog();
diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c index 4504608..08709f2 100644 --- a/tools/testing/selftests/bpf/progs/rbtree_fail.c +++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c
@@ -180,7 +180,7 @@ long rbtree_api_use_unchecked_remove_retval(void *ctx) } SEC("?tc") -__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") +__failure __msg("R2 type=scalar expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_api_add_release_unlock_escape(void *ctx) { struct node_data *n; @@ -204,7 +204,7 @@ long rbtree_api_add_release_unlock_escape(void *ctx) } SEC("?tc") -__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") +__failure __msg("R2 type=scalar expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_api_first_release_unlock_escape(void *ctx) { struct bpf_rb_node *res;
diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 338e438..e80f78f 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
@@ -118,8 +118,8 @@ long refcount_acquire_maybe_null(void *ctx) } SEC("?tc") -__failure __msg("R1 is neither owning or non-owning ref") -__msg("expects a pointer to a BPF-managed refcounted object, but R1 is a context pointer") +__failure __msg("R1 type=ctx expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") +__msg("type ctx, but this argument accepts ptr_, rcu_ptr_, ptr_, rcu_ptr_") long refcount_acquire_non_object(void *ctx) { return bpf_refcount_acquire(ctx) != NULL; @@ -159,8 +159,7 @@ long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx) SEC("?syscall") __failure -__msg("bpf_rbtree_remove can only take non-owning or refcounted " - "bpf_rb_node pointer") +__msg("R2 type=untrusted_ptr_ expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_remove_after_rcu_unlock(void *ctx) { struct map_value_rcu_graph *mapval; @@ -190,7 +189,7 @@ long rbtree_remove_after_rcu_unlock(void *ctx) } SEC("?syscall") -__failure __msg("R1 is neither owning or non-owning ref") +__failure __msg("R1 type=untrusted_ptr_ expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long refcount_acquire_after_rcu_unlock(void *ctx) { struct map_value_refcount_only *mapval;
diff --git a/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c b/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c index 330682a..8fd591b 100644 --- a/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c +++ b/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c
@@ -24,7 +24,7 @@ struct bpf_spin_lock lock __hidden SEC(".data.A"); struct bpf_res_spin_lock res_lock __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("point to map value or allocated object") +__failure __msg("R1 type=untrusted_ptr_ expected=map_value, ptr_") int res_spin_lock_arg(struct __sk_buff *ctx) { struct arr_elem *elem;
diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c index d96e99b..95e6a33 100644 --- a/tools/testing/selftests/bpf/progs/setget_sockopt.c +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c
@@ -61,8 +61,8 @@ static const struct sockopt_test sol_tcp_tests[] = { { .opt = TCP_NOTSENT_LOWAT, .new = 1314, .expected = 1314, }, { .opt = TCP_BPF_SOCK_OPS_CB_FLAGS, .new = BPF_SOCK_OPS_ALL_CB_FLAGS, .expected = BPF_SOCK_OPS_ALL_CB_FLAGS, }, - { .opt = TCP_BPF_DELACK_MAX, .new = 30000, .expected = 30000, }, - { .opt = TCP_BPF_RTO_MIN, .new = 30000, .expected = 30000, }, + { .opt = TCP_BPF_DELACK_MAX, .new = 40000, .expected = 40000, }, + { .opt = TCP_BPF_RTO_MIN, .new = 40000, .expected = 40000, }, { .opt = TCP_RTO_MAX_MS, .new = 2000, .expected = 2000, }, { .opt = 0, }, };
diff --git a/tools/testing/selftests/bpf/progs/skb_csum_complete.c b/tools/testing/selftests/bpf/progs/skb_csum_complete.c new file mode 100644 index 0000000..644731a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/skb_csum_complete.c
@@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +SEC("tc") +int change_tail_trim(struct __sk_buff *skb) +{ + /* + * Drops CHECKSUM_COMPLETE. skb->csum still covers the trimmed bytes, + * which hold the non-zero tcp.urg_ptr of pkt_v4, so it no longer + * matches the packet. + */ + if (bpf_skb_change_tail(skb, skb->len - 4, 0)) + return 1; + return 0; +} + +SEC("tc") +int change_tail_grow_write(struct __sk_buff *skb) +{ + __u8 val = 0xab; + + /* + * Drops CHECKSUM_COMPLETE. The new bytes are zero and do not change + * the sum, but the write below can no longer update skb->csum. + */ + if (bpf_skb_change_tail(skb, skb->len + 4, 0)) + return 1; + if (bpf_skb_store_bytes(skb, skb->len - 1, &val, sizeof(val), + BPF_F_RECOMPUTE_CSUM)) + return 2; + return 0; +} + +SEC("tc") +int store_no_recompute(struct __sk_buff *skb) +{ + __u8 val = 0xab; + + /* skb stays CHECKSUM_COMPLETE, but skb->csum is not updated. */ + if (bpf_skb_store_bytes(skb, skb->len - 1, &val, sizeof(val), 0)) + return 1; + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/stack_arg.c b/tools/testing/selftests/bpf/progs/stack_arg.c index 944e3bb..3e9ed37 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg.c +++ b/tools/testing/selftests/bpf/progs/stack_arg.c
@@ -21,7 +21,8 @@ struct { int timer_result; -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true;
diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index ad9d4bf..fff2e94 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c
@@ -3,20 +3,10 @@ #include <vmlinux.h> #include <bpf/bpf_helpers.h> -#include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" #if defined(__BPF_FEATURE_STACK_ARGUMENT) -SEC("tc") -__failure __msg("Unrecognized *(R11-8) type STRUCT") -int test_stack_arg_big(struct __sk_buff *skb) -{ - struct prog_test_big_arg s = { .a = 1, .b = 2 }; - - return bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s); -} - SEC("socket") __description("r11 in ALU instruction") __failure __msg("R11 is invalid") @@ -103,8 +93,9 @@ __naked void r11_store_zero_off(void) SEC("tc") __description("stack_arg_fail: not supported, dummy test") +__skip("stack_arg_fail: not supported") __success -int test_stack_arg_big(struct __sk_buff *skb) +int dummy_test(struct __sk_buff *skb) { return 0; }
diff --git a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c index 345f2da..b6b1a8a 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c
@@ -6,7 +6,8 @@ #include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true;
diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index bee2eee..c956477 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c
@@ -6,7 +6,8 @@ #include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) /* Force kfunc extern BTF generation for inline asm call below. @@ -73,7 +74,7 @@ __msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") __msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13") __msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") -__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx 11") __msg("mark_precise: frame0: regs= stack= before 9: (05) goto pc+1") __msg("mark_precise: frame0: regs= stack= before 8: (7a) *(u64 *)(r11 -8) = 4") @@ -94,7 +95,7 @@ __msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") __msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13 ") __msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") -__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx 11 ") __msg("mark_precise: frame0: regs= stack= before 10: (7a) *(u64 *)(r11 -8) = 6") __naked void stack_arg_precision_bpf2bpf(void) @@ -124,6 +125,7 @@ __naked void stack_arg_precision_bpf2bpf(void) SEC("socket") __description("stack_arg_precision: not supported, dummy test") +__skip("stack_arg_precision: not supported") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 8e8e133..1ae015c 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c
@@ -36,9 +36,15 @@ struct { } array SEC(".maps"); #define ENOSPC 28 +#define E2BIG 7 #define _STR "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define _X64 "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +/* 1024 bytes: truncated by bstr_printf, must return -E2BIG. */ +#define _BIG_STR (_X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64 \ + _X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64) int size; +int nmi_stream_prints; u64 fault_addr; void *arena_ptr; @@ -120,6 +126,48 @@ int stream_syscall(void *ctx) } SEC("syscall") +__success __retval(0) +int stream_empty(void *ctx) +{ + return bpf_stream_printk(BPF_STDOUT, ""); +} + +SEC("perf_event") +int stream_nmi(void *ctx) +{ + if (nmi_stream_prints) + return 0; + /* Retry on a later sample if the write failed, e.g. with -EBUSY. */ + if (bpf_stream_printk(BPF_STDOUT, "nmi")) + return 0; + nmi_stream_prints = 1; + return 0; +} + +SEC("syscall") +__success __retval(0) +int stream_oversize(void *ctx) +{ + int ret; + + ret = bpf_stream_printk(BPF_STDOUT, _BIG_STR); + if (ret != -E2BIG) + return ret ?: 1; + + /* The oversized output must not reduce the remaining stream capacity. */ + size = 0; + bpf_repeat(BPF_MAX_LOOPS) { + ret = bpf_stream_printk(BPF_STDOUT, _STR); + if (ret == -ENOSPC) + return size == 99954 ? 0 : 1; + if (ret) + return ret; + size += sizeof(_STR) - 1; + } + return 1; +} + +SEC("syscall") __arch_x86_64 __arch_arm64 __success __retval(0)
diff --git a/tools/testing/selftests/bpf/progs/stream_fail.c b/tools/testing/selftests/bpf/progs/stream_fail.c index 21428bb1..10ebb4a 100644 --- a/tools/testing/selftests/bpf/progs/stream_fail.c +++ b/tools/testing/selftests/bpf/progs/stream_fail.c
@@ -23,7 +23,7 @@ int stream_vprintk_scalar_arg(void *ctx) } SEC("syscall") -__failure __msg("R2 doesn't point to a const string") +__failure __msg("R2 type=ctx expected=map_value") int stream_vprintk_string_arg(void *ctx) { bpf_stream_vprintk(BPF_STDOUT, ctx, NULL, 0);
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c new file mode 100644 index 0000000..495faa0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c
@@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * Neither the callback nor the unrelated program refers to the arena. libbpf + * binds it to them all the same, so the callback can use the __arena argument + * the kernel hands it. + */ +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_autobind = { + .test_arena = (void *)test_arena_cb, +}; + +SEC("syscall") +int unrelated(void *ctx) +{ + return 0; +} + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (bpf_testmod_ops3_call_test_arena((u64 *)val)) + return 2; + if (*val != 42) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c new file mode 100644 index 0000000..a2218cd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c
@@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; +/* the register value a load of args->ptr produced in the callback */ +u64 __arena member_ptr_val; + +/* + * The header is shared with the module, so the members are u64 __arena * on + * this side too and the loads need no cast. The verifier derives the type of + * the loaded value from the module BTF, where the members carry the tag. + */ +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = args->ptr, *acq; + u64 raw; + + arena_touch++; + + /* the register right after the load, untouched by the compiler */ + asm volatile("%[raw] = *(u64 *)(%[args] + %[off])" + : [raw]"=r"(raw) + : [args]"r"(args), + [off]"i"(offsetof(struct bpf_testmod_arena_member_args, ptr))); + member_ptr_val = raw; + + *ptr += 1; + /* an acquire load of the member yields the same arena pointer */ + asm volatile("r1 = %[args];\n" + ".8byte 0x00000100000010db;\n" /* r0 = load_acquire((u64 *)(r1 + 0)) */ + "%[acq] = r0;" + : [acq]"=r"(acq) + : [args]"r"(args) + : "r0", "r1"); + if (acq != ptr) + return 0xbad; + *acq += 1; + if (args->null_ptr) + return 0xbad; + *args->nested.ptr += 1; + if (args->cookie != 0xc00c1e) + return 0xbad; + /* + * @faulting is a plain kernel pointer, so the member behind it is read + * with a probed load, which the JIT skips for a user address or which + * faults, and reads as NULL rather than as an arena pointer. + */ + if (args->faulting->ptr) + return 0xbad; + /* an arena pointer to kfuncs as well, the kfunc adds one */ + bpf_kfunc_arena_arg_test((u64 *)args->ptr); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member = { + .test_arena_member = (void *)test_arena_member_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr, its acquire load, nested.ptr and the kfunc each added one */ + if (*val != 45) + return 3; + + /* + * The callback saw (u32)(kaddr - kern_vm_start), the arena offset of + * val with the upper 32 bits clear, the same as an __arena argument. + */ + if (member_ptr_val != (u32)(u64)val) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c new file mode 100644 index 0000000..0b41478 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c
@@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reaches args->ptr on one path and args->cookie on the other, + * the branch on the unknown cookie value makes the verifier walk both. The + * rewrite is per insn, so the paths have to agree on what they load. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +__naked int mismatch_member(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = *(u64 *)(r6 + %[cookie_off]);" + "if r0 == 0 goto 1f;" + "r6 += %[cookie_off];" +"1: r0 = *(u64 *)(r6 + 0);" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena), + __imm_const(cookie_off, offsetof(struct bpf_testmod_arena_member_args, cookie)) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_member, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c new file mode 100644 index 0000000..ff6d4db --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c
@@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reads args->ptr on one path and the memory argument on the + * other. The verifier lets a load insn serve a kernel struct pointer and a + * memory pointer, see save_aux_ptr_type(), but the rewrite of the arena + * member would corrupt the value the other path loads. A global function + * is the only place a struct_ops program of this type gets a memory pointer. + * The loaded value goes nowhere: an ALU insn on it would be an arena + * pointer on one path and a scalar on the other, which the verifier rejects + * on its own before the fixup pass gets to the load. + */ +__noinline int load_either(unsigned long long *ctx __arg_ctx, u64 *mem __arg_nonnull) +{ + struct bpf_testmod_arena_member_args *args = (void *)ctx[0]; + void *p = args; + + if (args->cookie == 0) + p = mem; + /* one load insn, reached with both pointers */ + asm volatile("r0 = *(u64 *)(%[p] + 0)" :: [p]"r"(p) : "r0"); + return 0; +} + +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +int BPF_PROG(mismatch_mem, struct bpf_testmod_arena_member_args *args) +{ + u64 local = 0; + + asm volatile("" :: "r"(&arena)); + return load_either(ctx, &local); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_mem, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c new file mode 100644 index 0000000..cdd8ea7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c
@@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the object: the member cannot be translated into anything. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct bpf_testmod_arena_member_args " + "requires a program with an associated arena") +int BPF_PROG(no_arena_member, struct bpf_testmod_arena_member_args *args) +{ + return args->ptr != NULL; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)no_arena_member, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c new file mode 100644 index 0000000..d45666f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c
@@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * An offset into a union resolves to its first member, so a program reading + * raw there would get an arena pointer it did not ask for. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of bpf_testmod_arena_union is inside a union") +int BPF_PROG(union_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + return *args->u.ptr != 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)union_member, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c new file mode 100644 index 0000000..ff07619 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c
@@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The kernel owns the member, a program only reads it. A struct_ops program + * cannot store through its arguments at all, so the store is refused before + * the tag comes into play; the exchange test reaches the arena member check. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("only read is supported") +int BPF_PROG(write_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + args->ptr = NULL; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)write_member, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c new file mode 100644 index 0000000..0d81416 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c
@@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* An atomic that writes the member is neither a plain nor an acquire load. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct " + "bpf_testmod_arena_member_args can only be read with a plain or an acquire load") +__naked int xchg_member(void) +{ + asm volatile ( + "r1 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = 0;" + /* r0 = xchg_64((u64 *)(r1 + 0), r0): BPF_STX | BPF_ATOMIC | BPF_DW, imm BPF_XCHG */ + ".8byte 0x000000e1000001db;" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)xchg_member, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c new file mode 100644 index 0000000..09b4c74 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c
@@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> + +char _license[] SEC("license") = "GPL"; + +/* + * The module types as a vmlinux.h generated by a libbpf that does not render + * the arena tag with its address space shows them: an attribute on a pointer + * in the kernel address space, so the program has to cast the member to the + * arena address space before using it. + */ +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#define __nullable_tag __attribute__((btf_type_tag("nullable"))) + +struct bpf_testmod_arena_member_args { + __u64 __arena_tag *ptr; + __u64 __arena_tag __nullable_tag *null_ptr; + struct { + __u64 __arena_tag *ptr; + } nested; +} __attribute__((preserve_access_index)); + +struct bpf_testmod_ops3 { + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); +}; + +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; + +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_render_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = (u64 __arena *)args->ptr; + + arena_touch++; + *ptr += 1; + if (args->null_ptr) + return 0xbad; + *(u64 __arena *)args->nested.ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_render = { + .test_arena_member = (void *)test_arena_member_render_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr and nested.ptr each added one */ + if (*val != 43) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c index c62be15..9d290f2 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c
@@ -17,7 +17,6 @@ struct { } prog_array SEC(".maps"); SEC("struct_ops/test_refcounted_multi") -__failure __msg("program with __ref argument cannot tail call") int test_refcounted_multi(unsigned long long *ctx) { /* ctx[2] is used because the refcounted variable is the third argument */ @@ -29,7 +28,20 @@ int test_refcounted_multi(unsigned long long *ctx) return 0; } +__u64 trampoline_stack_arg9 = 0; + +SEC("struct_ops/test_trampoline_stack_args") +int BPF_PROG(test_trampoline_stack_args, int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9) +{ + trampoline_stack_arg9 = arg9; + + return 0; +} + SEC(".struct_ops.link") struct bpf_testmod_ops testmod_ref_acquire = { .test_refcounted_multi = (void *)test_refcounted_multi, + .test_trampoline_stack_args = (void *)test_trampoline_stack_args, };
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c new file mode 100644 index 0000000..929969e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c
@@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +/* an untrusted pointer to the argument struct, never set, for the verifier */ +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +SEC("struct_ops/test_nullable") +int BPF_PROG(test_nullable_cb, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + struct task_struct *t; + int ret = 0, *i, zero = 0; + + /* trusted, so good for a kfunc as is */ + t = bpf_task_acquire(args->task); + if (!t) + return -1; + bpf_task_release(t); + + /* trusted but __nullable, so a check has to come first */ + if (args->task_or_null) { + t = bpf_task_acquire(args->task_or_null); + if (!t) + return -1; + bpf_task_release(t); + ret = 1; + } + /* per-CPU and __nullable: a check has to come first, per_cpu_ptr() has none */ + if (args->pcpu_or_null) { + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + if (v->val != 0x5ca1ab1e) + return -1; + } + /* through a plain pointer the loads are probed, __nullable still asks for a check */ + t = args->self->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + /* a pointer to a scalar walks as untrusted memory, __nullable applies all the same */ + i = args->int_or_null; + if (i && *i != 7) + return -1; + /* through an untrusted pointer the member is probed too, and stays nullable */ + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (u) { + t = u->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + } + return ret; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable = { + .test_nullable = (void *)test_nullable_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ + /* one call without task_or_null, one with */ + return bpf_testmod_ops3_call_test_nullable() != 2; +}
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c new file mode 100644 index 0000000..c15b3a8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c
@@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* The member is trusted, but __nullable, so it needs a check before the kfunc. */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(unchecked_nullable, struct bpf_testmod_nullable_args *args) +{ + struct task_struct *t; + + t = bpf_task_acquire(args->task_or_null); + if (t) + bpf_task_release(t); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_nullable, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c new file mode 100644 index 0000000..6520151 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c
@@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The member is per-CPU and __nullable: per_cpu_ptr() shifts NULL like any + * other pointer, so a check has to come before the helper. + */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(unchecked_percpu, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + return v->val; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_percpu, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c new file mode 100644 index 0000000..c45f6d3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c
@@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * Through a plain pointer the member is probed, which tolerates NULL, so the + * tag asks for no check there, as the BTF_TYPE_SAFE_*_OR_NULL lists never did. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_plain, struct bpf_testmod_nullable_args *args) +{ + return args->self->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_plain, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c new file mode 100644 index 0000000..ff76874 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c
@@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * A pointer to a scalar walks as untrusted memory, which is probed, so the + * tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_scalar, struct bpf_testmod_nullable_args *args) +{ + return *args->int_or_null; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_scalar, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c new file mode 100644 index 0000000..9619991 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c
@@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +/* + * Through an untrusted pointer the member is probed, which tolerates NULL, so + * the tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_untrusted, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + int zero = 0; + + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (!u) + return 0; + return u->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_untrusted, +};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c index 1442728..c8cb35b 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c
@@ -4,6 +4,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" char _license[] SEC("license") = "GPL"; @@ -25,6 +26,44 @@ __noinline static int subprog1(int *a) return subprog2(a, b); } +/* + * A chain of 480-byte frames under test_2, so that its call chain exceeds + * the 2 KiB budget of JITs with large stacks as well as the 512 bytes + * allowed elsewhere. The compiler caps a single function at 512 bytes, and + * the buffers are volatile so that it cannot shrink them. + */ +__noinline static int subprog_deep4(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return a[10] + b[20]; +} + +__noinline static int subprog_deep3(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep4(a) + b[20]; +} + +__noinline static int subprog_deep2(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep3(a) + b[20]; +} + +__noinline static int subprog_deep1(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep2(a) + b[20]; +} + SEC("struct_ops") int BPF_PROG(test_1) @@ -41,11 +80,13 @@ int BPF_PROG(test_1) SEC("struct_ops") int BPF_PROG(test_2) { - /* stack size 400 bytes */ - int a[100] = {}; + /* stack size 476 bytes, over 2 KiB with the four 480-byte deep subprogs */ + volatile char buf[376] = {}; + int a[25] = {}; + __sink(buf[375]); a[10] = 3; - val_j = subprog1(a); + val_j = subprog1(a) + subprog_deep1(a); return 0; }
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c new file mode 100644 index 0000000..94a25a2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c
@@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +long val; + +/* On a private stack every frame gets the whole 2 KiB budget. */ +__used __naked +static long frame_2048_leaf(void) +{ + asm volatile (" \ + r1 = 30; \ + *(u64 *)(r10 - 2048) = r1; \ + r1 = 12; \ + *(u64 *)(r10 - 8) = r1; \ + r0 = *(u64 *)(r10 - 2048); \ + r1 = *(u64 *)(r10 - 8); \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* test_1 is the member bpf_testmod requests a private stack for */ +SEC("struct_ops") +__naked int test_1(void) +{ + asm volatile (" \ + r1 = 100; \ + *(u64 *)(r10 - 2048) = r1; \ + call frame_2048_leaf; \ + r1 = *(u64 *)(r10 - 2048); \ + r0 += r1; \ + r1 = %[val] ll; \ + *(u64 *)(r1 + 0) = r0; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(val) + : __clobber_all); +} + +SEC(".struct_ops") +struct bpf_testmod_ops3 testmod_1 = { + .test_1 = (void *)test_1, +};
diff --git a/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c new file mode 100644 index 0000000..ca764411 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c
@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +SEC("freplace") +int subprog_freplace(struct __sk_buff *skb) +{ + bpf_tail_call_static(skb, &jmp_table, 0); + return 0; +} + +SEC("freplace") +int entry_freplace(struct __sk_buff *skb) +{ + return 0; +} + +char __license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tailcall_large_stack.c b/tools/testing/selftests/bpf/progs/tailcall_large_stack.c new file mode 100644 index 0000000..977197d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_large_stack.c
@@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +/* The tail call target sets up a 2 KiB frame of its own and uses both of its ends. */ +SEC("tc") +__naked int classifier_0(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 2048) = r1; \ + r1 = 7; \ + *(u64 *)(r10 - 8) = r1; \ + r0 = *(u64 *)(r10 - 2048); \ + r1 = *(u64 *)(r10 - 8); \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The frame of the subprog doing the tail call is unwound by it, so it may be + * large; only the frames of its callers stay behind and are limited to 256 + * bytes in total. Returns 1 when the tail call falls through. + */ +__used __naked +static int subprog_tail(void) +{ + asm volatile (" \ + r2 = 1; \ + *(u64 *)(r10 - 1536) = r2; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 1; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +SEC("tc") +__naked int entry(void) +{ + asm volatile (" \ + r2 = 2; \ + *(u64 *)(r10 - 240) = r2; \ + call subprog_tail; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index f96b0c1..12c8ac6 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c
@@ -50,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("R1 is fp expected STRUCT task_struct") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; @@ -179,7 +179,7 @@ int BPF_PROG(task_kfunc_release_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(task_kfunc_release_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired = (struct task_struct *)&clone_flags; @@ -225,7 +225,7 @@ int BPF_PROG(task_kfunc_release_null, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_unacquired, struct task_struct *task, u64 clone_flags) { /* Cannot release trusted task pointer which was not acquired. */ @@ -333,7 +333,7 @@ int BPF_PROG(task_access_comm2, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("write into memory") +__failure __msg("only read is supported") int BPF_PROG(task_access_comm3, struct task_struct *task, u64 clone_flags) { bpf_probe_read_kernel(task->comm, 16, task->comm); @@ -353,7 +353,7 @@ int BPF_PROG(task_access_comm4, struct task_struct *task, const char *buf, bool } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flags) { struct task_struct *local;
diff --git a/tools/testing/selftests/bpf/progs/task_work_busy.c b/tools/testing/selftests/bpf/progs/task_work_busy.c new file mode 100644 index 0000000..9960011 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/task_work_busy.c
@@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ +#include <vmlinux.h> +#include <stdbool.h> +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +char _license[] SEC("license") = "GPL"; + +extern bool CONFIG_PREEMPT_RT __kconfig __weak; + +/* + * Bounds for the hardirq busy-wait that holds the scheduling handler + * inside the real window: task_work_add() has published the callback, + * but the handler's tail state transition has not run yet. + * irq_work_single() clears BUSY only after the handler returns, so + * while this wait runs, the context's scheduling irq_work is BUSY, + * even though the first callback may already have completed and + * returned the context to STANDBY. + * + * The wait is bounded both by a bpf_ktime_get_ns() check and by an + * iteration budget. The time bound is checked against a monotonic + * clock: host vCPU preemption can stretch the wall-clock duration of + * the window, which only makes the window longer, never shorter. + */ +#define BUSY_WAIT_NS 4000000ULL +#define BUSY_WAIT_ITERS 1000000ULL + +int trigger_tid; +int target_tid; + +/* + * High 32 bits of the result slots hold the sequence, low 32 bits the + * signed error. BUSY_WINDOW low bits: 1 open, 2 the in-window attempt + * was seen before the handler returned, 3 the wait bound was reached + * without it. Must match the test. + */ +enum task_work_busy_status { + BUSY_ARM_SEQ, + BUSY_WINDOW, + BUSY_ACK, + BUSY_GEN1_RESULT, + BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, + BUSY_CB2_COUNT, + BUSY_CB2_SEQ, + BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + BUSY_STATUS_MAX, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, BUSY_STATUS_MAX); + __type(key, int); + __type(value, __s64); +} status SEC(".maps"); + +struct task_work_busy_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +/* + * max_entries must cover TASK_WORK_BUSY_MAX_ATTEMPTS in the test: each + * attempt schedules into its own fresh value. + */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 128); + __type(key, int); + __type(value, struct task_work_busy_value); +} hmap SEC(".maps"); + +static __always_inline void set_status(int key, __s64 value) +{ + __s64 *slot; + + slot = bpf_map_lookup_elem(&status, &key); + if (slot) + *slot = value; +} + +static __always_inline __s64 get_status(int key) +{ + __s64 *slot = bpf_map_lookup_elem(&status, &key); + + return slot ? *slot : 0; +} + +struct busy_wait_ctx { + __u64 seq; + __u64 start; + bool acked; +}; + +/* bpf_loop() callback: return 0 continues the loop, return 1 stops it. */ +static long busy_wait_cb(__u32 i, void *pctx) +{ + struct busy_wait_ctx *w = pctx; + + if (((__u64)get_status(BUSY_ACK) >> 32) == w->seq) { + w->acked = true; + return 1; + } + if (bpf_ktime_get_ns() - w->start > BUSY_WAIT_NS) + return 1; + return 0; +} + +static int busy_cb_gen1(struct bpf_map *map, void *key, void *value) +{ + struct task_work_busy_value *work = value; + __u64 we = (__u64)get_status(BUSY_WINDOW) >> 32; + + /* + * Backstop for deliveries from abandoned attempts: accept only + * while no window for a different sequence is published. The + * test waits for the old workers to disappear from PID lookup, + * after their exit task_work, before resetting the shared state. + * This callback may run before fexit publishes the window state. + */ + if (we && we != (__u64)work->seq) + return 0; + set_status(BUSY_CB1_SEQ, work->seq); + return 0; +} + +static int busy_cb_gen2(struct bpf_map *map, void *key, void *value) +{ + struct task_work_busy_value *work = value; + __s64 *count; + int key2 = BUSY_CB2_COUNT; + + /* + * Count only deliveries belonging to the sequence whose window is + * (or was) published: a late delivery from an abandoned attempt + * must not satisfy a later exactly-once check. + */ + if ((__u64)work->seq != ((__u64)get_status(BUSY_WINDOW) >> 32)) + return 0; + set_status(BUSY_CB2_SEQ, work->seq); + count = bpf_map_lookup_elem(&status, &key2); + if (count) + __sync_fetch_and_add(count, 1); + return 0; +} + +SEC("fexit/task_work_add") +int BPF_PROG(busy_fexit_task_work_add, struct task_struct *task, + struct callback_head *work, int notify, int retval) +{ + struct busy_wait_ctx w = {}; + __u64 seq, elapsed; + + seq = (__u64)get_status(BUSY_ARM_SEQ); + if (!seq) + return 0; + + /* + * One-shot arm. Only this test's own first-round add may consume + * it and hold the window: a successful TWA_SIGNAL add of the + * bpf_task_work callback, issued from the trigger thread's CPU by + * the self-IPI'd scheduling handler and targeting the target + * thread. Everything else leaves the arm untouched. + */ + if (retval != 0 || notify != TWA_SIGNAL || + BPF_CORE_READ(task, pid) != target_tid) + return 0; + if ((__u32)bpf_get_current_pid_tgid() != (__u32)trigger_tid) + return 0; + + set_status(BUSY_ARM_SEQ, 0); + set_status(BUSY_WINDOW, (seq << 32) | 1); + + w.seq = seq; + w.start = bpf_ktime_get_ns(); + bpf_loop(BUSY_WAIT_ITERS, busy_wait_cb, &w, 0); + elapsed = bpf_ktime_get_ns() - w.start; + + /* + * Publish elapsed time for logging after worker cleanup. Window + * validity depends on seeing ACK here, not on this diagnostic. + */ + set_status(BUSY_LAST_WAIT_NS, (__s64)elapsed); + if (w.acked) + set_status(BUSY_WINDOW, (seq << 32) | 2); + else + set_status(BUSY_WINDOW, (seq << 32) | 3); + return 0; +} + +/* + * Attempt the second scheduling round on the same context, targeting the + * trigger thread. The attempt uses the value keyed by its own sequence, + * so it can never touch another attempt's context. Returns true if a + * terminal result was published into res_key; a transient -EBUSY is left + * unpublished for the caller's retry policy, mirroring the race test's + * convention. + */ +static __always_inline bool busy_try_gen2(__u64 seq, int res_key) +{ + struct task_work_busy_value *work; + struct task_struct *task; + int key = (int)seq, err; + + task = bpf_task_from_pid(trigger_tid); + if (!task) { + set_status(res_key, ((__s64)seq << 32) | (__u32)-ESRCH); + return true; + } + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + bpf_task_release(task); + set_status(res_key, ((__s64)seq << 32) | (__u32)-ENOENT); + return true; + } + work->seq = seq; + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + busy_cb_gen2); + bpf_task_release(task); + if (err == -EBUSY) + return false; + set_status(res_key, ((__s64)seq << 32) | (__u32)err); + return true; +} + +SEC("tracepoint/syscalls/sys_enter_getppid") +int busy_sched_work(void *ctx) +{ + __u32 tid = (__u32)bpf_get_current_pid_tgid(); + struct task_work_busy_value *work; + struct task_struct *task; + __u64 window, seq; + int key, err; + + /* Also keep the kconfig extern in object BTF for the userspace skip. */ + if (CONFIG_PREEMPT_RT) + return 0; + + if (tid == (__u32)trigger_tid) { + /* + * Trigger thread: schedule the first round once per armed + * sequence, targeting the target thread. The kfunc queues + * the scheduling irq_work and returns; the handler then + * parks inside the fexit window above. A fresh value has no + * expected BUSY source, so publish every scheduling result. + */ + seq = (__u64)get_status(BUSY_ARM_SEQ); + if (!seq || + ((__u64)get_status(BUSY_GEN1_RESULT) >> 32) == seq) + return 0; + key = (int)seq; + task = bpf_task_from_pid(target_tid); + if (!task) { + set_status(BUSY_GEN1_RESULT, + ((__s64)seq << 32) | (__u32)-ESRCH); + set_status(BUSY_ARM_SEQ, 0); + return 0; + } + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + bpf_task_release(task); + set_status(BUSY_GEN1_RESULT, + ((__s64)seq << 32) | (__u32)-ENOENT); + set_status(BUSY_ARM_SEQ, 0); + return 0; + } + work->seq = seq; + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + busy_cb_gen1); + bpf_task_release(task); + set_status(BUSY_GEN1_RESULT, ((__s64)seq << 32) | (__u32)err); + return 0; + } + + if (tid != (__u32)target_tid) + return 0; + + window = (__u64)get_status(BUSY_WINDOW); + seq = window >> 32; + if (!seq || (__u32)window == 0) + return 0; + /* + * The first callback must have completed for this sequence. Being + * inside the target thread's own subsequent syscall proves the + * callback wrapper has returned and the context is STANDBY, which + * is stronger than observing the completion marker from another + * thread. + */ + if ((__u64)get_status(BUSY_CB1_SEQ) != seq) + return 0; + + if ((__u64)get_status(BUSY_ACK) >> 32 != seq) { + /* + * The in-window attempt, exactly once per sequence. On a + * fixed kernel it must be rejected with -EBUSY: the context + * is STANDBY, the map and program references are alive and + * no other scheduler exists, so only the BUSY gate can + * reject here. + */ + if (!busy_try_gen2(seq, BUSY_ACK)) + set_status(BUSY_ACK, + ((__s64)seq << 32) | (__u32)-EBUSY); + return 0; + } + + /* + * Recovery, gated by the test: the fexit program releasing the + * window does not mean the whole old handler has returned and + * BUSY has cleared yet, so retry until the schedule succeeds. + */ + if ((__u64)get_status(BUSY_RECOVER_SEQ) != seq) + return 0; + if ((__s32)get_status(BUSY_ACK) == 0) + return 0; /* scheduled in-window; nothing to recover */ + if (((__u64)get_status(BUSY_GEN2_RESULT) >> 32) == seq) + return 0; /* terminal recovery result published */ + busy_try_gen2(seq, BUSY_GEN2_RESULT); + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/task_work_fail.c b/tools/testing/selftests/bpf/progs/task_work_fail.c index 3186e7b..bc56bda 100644 --- a/tools/testing/selftests/bpf/progs/task_work_fail.c +++ b/tools/testing/selftests/bpf/progs/task_work_fail.c
@@ -58,7 +58,7 @@ int mismatch_map(struct pt_regs *args) } SEC("perf_event") -__failure __msg("R2 doesn't point to a map value") +__failure __msg("R2 type=fp expected=map_value") int no_map_task_work(struct pt_regs *args) { struct task_struct *task;
diff --git a/tools/testing/selftests/bpf/progs/task_work_race.c b/tools/testing/selftests/bpf/progs/task_work_race.c new file mode 100644 index 0000000..c94eca5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/task_work_race.c
@@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ +#include <vmlinux.h> +#include <stdbool.h> +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +#define TASK_WORK_RACE_MAX_SEQ 2048 + +enum task_work_race_status { + RACE_ARM_SEQ, + RACE_READY_SEQ, + /* High 32 bits hold the sequence, low 32 bits the signed error. */ + RACE_RESULT, + RACE_STATUS_MAX, +}; + +int trigger_tid; +int target_tid; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, RACE_STATUS_MAX); + __type(key, int); + __type(value, __s64); +} status SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, TASK_WORK_RACE_MAX_SEQ); + __type(key, int); + __type(value, __u64); +} completed SEC(".maps"); + +struct task_work_race_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 4); + __type(key, int); + __type(value, struct task_work_race_value); +} hmap SEC(".maps"); + +static __always_inline void set_status(int key, __s64 value) +{ + __s64 *slot; + + slot = bpf_map_lookup_elem(&status, &key); + if (slot) + *slot = value; +} + +static int process_work(struct bpf_map *map, void *key, void *value) +{ + struct task_work_race_value *work = value; + __u64 *done; + int seq = work->seq; + + if (seq <= 0 || seq >= TASK_WORK_RACE_MAX_SEQ) + return 0; + done = bpf_map_lookup_elem(&completed, &seq); + if (done) + *done = 1; + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_getppid") +int race_sched_work(void *ctx) +{ + struct task_work_race_value *work; + struct task_struct *task; + __s64 *arm, *result; + __u32 tid = (__u32)bpf_get_current_pid_tgid(); + int key = 0, err; + __u32 seq; + + if (tid != trigger_tid) + return 0; + + key = RACE_ARM_SEQ; + arm = bpf_map_lookup_elem(&status, &key); + key = RACE_RESULT; + result = bpf_map_lookup_elem(&status, &key); + if (!arm || !result || *arm <= 0 || + (__u32)((__u64)*result >> 32) == *arm) + return 0; + seq = *arm; + + task = bpf_task_from_pid(target_tid); + if (!task) { + err = -ESRCH; + goto out_done; + } + + key = 0; + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + err = -ENOENT; + goto out_task; + } + + work->seq = seq; + set_status(RACE_READY_SEQ, seq); + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + process_work); + if (err == -EBUSY) { + bpf_task_release(task); + return 0; + } + +out_task: + bpf_task_release(task); +out_done: + /* Publish the sequence and its error in one store. */ + *result = ((__u64)seq << 32) | (__u32)err; + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/tcp_autolowat.c b/tools/testing/selftests/bpf/progs/tcp_autolowat.c new file mode 100644 index 0000000..8a55f3c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_autolowat.c
@@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ +#include "vmlinux.h" + +#include <string.h> +#include <limits.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +#include "bpf_tracing_net.h" + +#define SOL_BPF 0xdeadbeef +#define BPF_TCP_AUTOLOWAT 0x8badf00d + +//#define DEBUG /* For verbose output. */ + +struct rpc_descriptor { + u32 header_len; + u32 payload_len; +}; + +#define RPC_DESC_SIZE (sizeof(struct rpc_descriptor)) +#define MAX_RPC_DESC_PER_SKB 100 + +struct tcp_autolowat_cb { + /* Don't put this field at the end; BPF verifier complains. */ + char rpc_desc_buf[RPC_DESC_SIZE]; + u32 rpc_desc_seq; + u32 rpc_end_seq; +#ifdef DEBUG + u32 isn; +#endif + u8 rpc_desc_buff_len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tcp_autolowat_cb); +} tcp_autolowat_map SEC(".maps"); + +char test_name[64]; + +#ifdef DEBUG +#define LOG(str, ...) \ + bpf_printk("%s: " str, test_name, ##__VA_ARGS__) +#else +#define LOG(...) +#endif + +#define SEQ(val) \ + (val - cb->isn) +#define TP_SEQ(field) \ + (tp->field - cb->isn) +#define CB_SEQ(field) \ + (cb->field - cb->isn) + +static int tcp_parse_descriptor(struct tcp_autolowat_cb *cb, + struct bpf_dynptr *dptr, + u32 seq, u32 end_seq) +{ + struct rpc_descriptor *rpc_desc; + u32 rpc_copied_seq; + u64 copy_len; /* u32 should work, but not for no_alu32 :/ */ + u64 rpc_len; + int err; + + rpc_copied_seq = cb->rpc_desc_seq + cb->rpc_desc_buff_len; + + if (before(cb->rpc_desc_seq + RPC_DESC_SIZE, end_seq)) + copy_len = RPC_DESC_SIZE - cb->rpc_desc_buff_len; + else + copy_len = end_seq - rpc_copied_seq; + + if (copy_len == 0) + goto disable; /* FIN. */ + if (copy_len > RPC_DESC_SIZE) + goto disable; /* always false, only for verifier. */ + if (cb->rpc_desc_buff_len >= RPC_DESC_SIZE) + goto disable; /* always false, only for verifier. */ + + err = bpf_dynptr_read(cb->rpc_desc_buf + cb->rpc_desc_buff_len, + copy_len, dptr, rpc_copied_seq - seq, 0); + if (err) + goto disable; + + cb->rpc_desc_buff_len += copy_len; + + if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) { + LOG("Copied %d bytes: rpc_desc_buff_len: %u", copy_len, cb->rpc_desc_buff_len); + goto partial; + } + + rpc_desc = (struct rpc_descriptor *)cb->rpc_desc_buf; + rpc_len = RPC_DESC_SIZE + rpc_desc->header_len + rpc_desc->payload_len; + + if (rpc_len > INT_MAX) + goto disable; + + cb->rpc_end_seq = cb->rpc_desc_seq + rpc_len; + + LOG("Copied full descriptor: rpc_desc_seq: %u, rpc_end_seq: %u, header_len: %u, payload_len: %u", + CB_SEQ(rpc_desc_seq), CB_SEQ(rpc_end_seq), + rpc_desc->header_len, rpc_desc->payload_len); + + return 0; +disable: + return -1; +partial: + return 1; +} + +static void tcp_set_autolowat(struct tcp_autolowat_cb *cb, + struct sock *sk) +{ + struct tcp_sock *tp = (struct tcp_sock *)sk; + u32 val; /* To handle wraparound. */ + + LOG("Setting rcvlowat: tp->copied_seq: %u, rpc_desc_seq: %u, rpc_end_seq: %u, rpc_desc_buff_len: %u", + TP_SEQ(copied_seq), CB_SEQ(rpc_desc_seq), + CB_SEQ(rpc_end_seq), cb->rpc_desc_buff_len); + + if (before(tp->copied_seq, cb->rpc_desc_seq)) + val = cb->rpc_desc_seq - tp->copied_seq; + else if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) + val = RPC_DESC_SIZE; + else + val = cb->rpc_end_seq - tp->copied_seq; + + if (val != tp->inet_conn.icsk_inet.sk.sk_rcvlowat) { + bpf_tcp_ops_set_rcvlowat(sk, val); + + LOG("Set rcvlowat: expected: %u, actual: %d\n", + val, tp->inet_conn.icsk_inet.sk.sk_rcvlowat); + } else { + LOG("No need to set rcvlowat: %u\n", val); + } +} + +static void tcp_disable_autolowat(struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, 0, BPF_TCP_OPS_FLAG_RCVQ); + + bpf_tcp_ops_set_rcvlowat(sk, 1); + + LOG("Disabled autolowat"); +} + +static void tcp_do_autolowat(struct tcp_autolowat_cb *cb, + struct sock *sk, struct sk_buff *skb) +{ + struct bpf_dynptr dptr; + struct tcp_skb_cb *tcb; + u32 seq, end_seq; + int ret = 0, i; + + if (bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &dptr)) { + ret = -1; + goto update; + } + + tcb = bpf_core_cast(skb->cb, struct tcp_skb_cb); + seq = tcb->seq; + end_seq = tcb->end_seq - !!(tcb->tcp_flags & TCPHDR_FIN); + + LOG("Start parsing skb: seq: %u, end_seq: %u, len: %u, rpc_desc_seq: %u, rpc_end_seq: %u, rpc_buff_len: %u", + SEQ(seq), SEQ(end_seq), end_seq - seq, + CB_SEQ(rpc_desc_seq), CB_SEQ(rpc_end_seq), cb->rpc_desc_buff_len); + + if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) { + ret = tcp_parse_descriptor(cb, &dptr, seq, end_seq); + if (ret) + goto update; + } + + i = 0; + + while (1) { + if (i++ > MAX_RPC_DESC_PER_SKB) { + ret = -1; + break; + } + + if (after(cb->rpc_end_seq, end_seq)) { + LOG("No more descriptor: rpc_end_seq: %u, end_seq: %u", + CB_SEQ(rpc_end_seq), SEQ(end_seq)); + break; + } + + cb->rpc_desc_seq = cb->rpc_end_seq; + cb->rpc_desc_buff_len = 0; + + if (cb->rpc_end_seq == end_seq) + break; + + LOG("Found next descriptor: rpc_end_seq: %u, end_seq: %u, len: %u", + CB_SEQ(rpc_end_seq), SEQ(end_seq), end_seq - cb->rpc_end_seq); + + ret = tcp_parse_descriptor(cb, &dptr, seq, end_seq); + if (ret) + break; + } + +update: + if (ret >= 0) + tcp_set_autolowat(cb, sk); + else + tcp_disable_autolowat(sk); +} + +SEC("struct_ops") +void BPF_PROG(tcp_autolowat_enqueue_rcvq, struct sock *sk, struct sk_buff *skb) +{ + struct tcp_autolowat_cb *cb; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, sk, 0, 0); + if (!cb) + return; + + tcp_do_autolowat(cb, sk, skb); +} + +SEC("struct_ops") +void BPF_PROG(tcp_autolowat_dequeue_rcvq, struct sock *sk) +{ + struct tcp_autolowat_cb *cb; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, sk, 0, 0); + if (!cb) + return; + + tcp_set_autolowat(cb, sk); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_autolowat_ops = { + .enqueue_rcvq = (void *)tcp_autolowat_enqueue_rcvq, + .dequeue_rcvq = (void *)tcp_autolowat_dequeue_rcvq, +}; + +static int tcp_init_autolowat_cb(struct bpf_tcp_sock *btp) +{ + struct tcp_autolowat_cb *cb; + struct tcp_sock *tp; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, btp, 0, + BPF_SK_STORAGE_GET_F_CREATE); + if (!cb) + return -1; + + tp = bpf_core_cast(btp, struct tcp_sock); + + cb->rpc_desc_seq = tp->copied_seq; + cb->rpc_end_seq = tp->copied_seq; +#ifdef DEBUG + cb->isn = tp->copied_seq; +#endif + + return bpf_tcp_ops_set_flags((struct tcp_sock *)btp, + BPF_TCP_OPS_FLAG_RCVQ, 0); +} + +SEC("cgroup/setsockopt") +int tcp_autolowat_setsockopt(struct bpf_sockopt *ctx) +{ + void *optval_end = ctx->optval_end; + int *optval = ctx->optval; + struct bpf_tcp_sock *btp; + + if (ctx->level != SOL_BPF || ctx->optname != BPF_TCP_AUTOLOWAT) + goto out; + + if (optval + 1 > optval_end) + return 0; /* -EPERM */ + + btp = bpf_tcp_sock(ctx->sk); + if (!btp) + goto out; + + if (*optval && tcp_init_autolowat_cb(btp)) + return 0; /* -EPERM */ + + /* + * BPF has consumed this option, don't call kernel + * setsockopt handler. + */ + ctx->optlen = -1; +out: + return 1; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_global_func1.c b/tools/testing/selftests/bpf/progs/test_global_func1.c index fc69ff1..f0eca28 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func1.c +++ b/tools/testing/selftests/bpf/progs/test_global_func1.c
@@ -48,8 +48,73 @@ int f3(int val, struct __sk_buff *skb, int var) } SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 3 calls is") int global_func1(struct __sk_buff *skb) { return f0(1, skb) + f1(skb) + f2(2, skb) + f3(3, skb, 4); } + +/* + * A chain of five frames that stay under 512 bytes each but add up to more + * than the 2 KiB budget of JITs with large stacks; the chain also exceeds + * 512 bytes after two frames, so it is rejected everywhere. + */ +#define MAX_STACK_LARGE 480 + +__attribute__ ((noinline)) +int g0(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return skb->len; +} + +__attribute__ ((noinline)) +int g1(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g0(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g2(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g1(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g3(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g2(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g4(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g3(skb) + skb->len; +} + +SEC("tc") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +int global_func1_deep(struct __sk_buff *skb) +{ + return g4(skb); +}
diff --git a/tools/testing/selftests/bpf/progs/test_global_func5.c b/tools/testing/selftests/bpf/progs/test_global_func5.c index 257c056..4573ecd 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func5.c +++ b/tools/testing/selftests/bpf/progs/test_global_func5.c
@@ -26,7 +26,7 @@ int f3(int val, struct __sk_buff *skb) } SEC("tc") -__failure __msg("expects pointer to ctx") +__failure __msg("R2 type=fp expected=ctx") int global_func5(struct __sk_buff *skb) { return f1(skb) + f2(2, skb) + f3(3, skb);
diff --git a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c index 1b634b5..edb8a22 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c +++ b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c
@@ -67,12 +67,30 @@ int XCAT(f, n)(unsigned long a) \ #define F_31 F_30 FN(31, 30) #define F_32 F_31 FN(32, 31) +/* Same, with a 480-byte frame, to exceed the 2 KiB budget of large stacks. */ +#define FNB(n, prev) \ +__attribute__((noinline)) \ +int XCAT(f, n)(unsigned long a) \ +{ \ + volatile char buf[480] = {}; \ + volatile long b = XCAT(f, prev)(a - 1); \ + if (!b) \ + return 0; \ + return b + buf[479] + 1; \ +} + +#define F_33 F_32 FNB(33, 32) +#define F_34 F_33 FNB(34, 33) +#define F_35 F_34 FNB(35, 34) +#define F_36 F_35 FNB(36, 35) +#define F_37 F_36 FNB(37, 36) + #define CAT2(a, b) a ## b #define XCAT2(a, b) CAT2(a, b) #define F(n) XCAT2(F_, n) -F(32) +F(37) /* Ensure that even 32 levels deep, the function verifies. */ SEC("syscall") @@ -88,8 +106,21 @@ int global_func_deep_stack_success(struct __sk_buff *skb) * the size. */ SEC("syscall") +__load_if_no_large_stack() __failure __msg("combined stack size of 34 calls") int global_func_deep_stack_fail(struct __sk_buff *skb) { return f32(123); } + +/* + * Five 480-byte frames on top of the chain: 5 * 480 + 33 * 16 = 2928 bytes, + * more than the 2 KiB budget of JITs with large stacks, and more than 512 + * bytes after the second frame everywhere else. + */ +SEC("syscall") +__failure __msg("combined stack size of {{[0-9]+}} calls") +int global_func_deep_stack_fail_large(struct __sk_buff *skb) +{ + return f37(123); +}
diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index 5dc21b3..2765cd4 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c
@@ -8,9 +8,12 @@ int loong SEC(".percpu.looooooooong"); int data3 SEC(".data.percpu"); int data2 SEC(".percpu.data"); +/* Used for testing bpf_map__set_value_size(). */ +int arr[1] SEC(".percpu.arr"); +int arr_sum; + int run; -/* cpu_id as array to verify map value resizing. */ -int cpu_id[1] SEC(".percpu"); +int cpu_id SEC(".percpu"); int data SEC(".percpu") = -1; int nums[7] SEC(".percpu"); bool set SEC(".percpu") = false; @@ -34,7 +37,8 @@ int update_percpu_data(void *ctx) data = 1; run++; set = true; - cpu_id[0] = bpf_get_smp_processor_id(); + cpu_id = bpf_get_smp_processor_id(); + arr_sum += arr[0]; return 0; }
diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c index bf48fc4..f7a83e5 100644 --- a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c +++ b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c
@@ -40,7 +40,7 @@ int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size, } SEC("?lsm.s/bpf") -__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 type=map_value expected=fp, dynptr_ptr") int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size, bool kernel) { static struct bpf_dynptr val;
diff --git a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c deleted file mode 100644 index cbe4284..0000000 --- a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c +++ /dev/null
@@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young <sean@mess.org> - -#include <linux/bpf.h> -#include <linux/lirc.h> -#include <bpf/bpf_helpers.h> - -SEC("lirc_mode2") -int bpf_decoder(unsigned int *sample) -{ - if (LIRC_IS_PULSE(*sample)) { - unsigned int duration = LIRC_VALUE(*sample); - - if (duration & 0x1000) - bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0); - if (duration & 0x2000) - bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff, - duration & 0xff); - } - - return 0; -} - -char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c index 4cfa42a..c5c6430 100644 --- a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c +++ b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c
@@ -29,7 +29,7 @@ int BPF_PROG(unix_listen, struct socket *sock, int backlog) len = unix_sk->addr->len - sizeof(short); path[0] = '@'; for (i = 1; i < len; i++) { - if (i >= (int)sizeof(struct sockaddr_un)) + if (i >= (int)sizeof(unix_sk->addr->name->sun_path)) break; path[i] = unix_sk->addr->name->sun_path[i];
diff --git a/tools/testing/selftests/bpf/progs/test_snprintf.c b/tools/testing/selftests/bpf/progs/test_snprintf.c index 8fda075..48a0025 100644 --- a/tools/testing/selftests/bpf/progs/test_snprintf.c +++ b/tools/testing/selftests/bpf/progs/test_snprintf.c
@@ -57,8 +57,8 @@ int handler(const void *ctx) sym_ret = BPF_SNPRINTF(sym_out, sizeof(sym_out), "%ps %pS %pB", &schedule, &schedule, &schedule); /* Kernel pointers */ - addr_ret = BPF_SNPRINTF(addr_out, sizeof(addr_out), "%pK %px %p", - 0, 0xFFFF00000ADD4E55, 0xFFFF00000ADD4E55); + addr_ret = BPF_SNPRINTF(addr_out, sizeof(addr_out), "%px %p", + 0xFFFF00000ADD4E55, 0xFFFF00000ADD4E55); /* Strings and single-byte character embedding */ str_ret = BPF_SNPRINTF(str_out, sizeof(str_out), "%s % 9c %+2c %-3c %04c %0c %+05s", str1, 'a', 'b', 'c', 'd', 'e', longstr);
diff --git a/tools/testing/selftests/bpf/progs/test_tc_edt.c b/tools/testing/selftests/bpf/progs/test_tc_edt.c index 4f6f031..69f4b16 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_edt.c +++ b/tools/testing/selftests/bpf/progs/test_tc_edt.c
@@ -24,6 +24,7 @@ struct { } flow_map SEC(".maps"); __uint64_t target_rate; +__uint64_t tx_start_ns; static inline int throttle_flow(struct __sk_buff *skb) { @@ -62,7 +63,8 @@ static inline int throttle_flow(struct __sk_buff *skb) return TC_ACT_OK; } -static inline int handle_tcp(struct __sk_buff *skb, struct tcphdr *tcp) +static inline int handle_tcp(struct __sk_buff *skb, struct tcphdr *tcp, + uint32_t hdr_len) { void *data_end = (void *)(long)skb->data_end; @@ -70,8 +72,12 @@ static inline int handle_tcp(struct __sk_buff *skb, struct tcphdr *tcp) if ((void *)(tcp + 1) > data_end) return TC_ACT_SHOT; - if (tcp->source == bpf_htons(9000)) + if (tcp->source == bpf_htons(9000)) { + /* Note when the first packet with data goes out */ + if (!tx_start_ns && skb->len > hdr_len + tcp->doff * 4) + tx_start_ns = bpf_ktime_get_ns(); return throttle_flow(skb); + } return TC_ACT_OK; } @@ -94,7 +100,8 @@ static inline int handle_ipv4(struct __sk_buff *skb) return TC_ACT_SHOT; if (iph->protocol == IPPROTO_TCP) - return handle_tcp(skb, (struct tcphdr *)(((void *)iph) + ihl)); + return handle_tcp(skb, (struct tcphdr *)(((void *)iph) + ihl), + sizeof(struct ethhdr) + ihl); return TC_ACT_OK; }
diff --git a/tools/testing/selftests/bpf/progs/test_tc_link.c b/tools/testing/selftests/bpf/progs/test_tc_link.c index 630f12e..2a91793 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_link.c +++ b/tools/testing/selftests/bpf/progs/test_tc_link.c
@@ -4,6 +4,8 @@ #include <linux/bpf.h> #include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/ip.h> #include <linux/stddef.h> #include <linux/if_packet.h> #include <bpf/bpf_endian.h> @@ -119,11 +121,19 @@ SEC("tc/egress") int tc8(struct __sk_buff *skb) { struct net_device *dev = BPF_CORE_READ((struct sk_buff *)skb, dev); + struct iphdr iph = {}; + if (skb->protocol != __bpf_constant_htons(ETH_P_IP)) + goto out; + if (bpf_skb_load_bytes(skb, sizeof(struct ethhdr), &iph, sizeof(iph))) + goto out; + if (iph.protocol != IPPROTO_ICMP) + goto out; seen_tc8 = true; mark = skb->mark; prio = skb->priority; headroom = BPF_CORE_READ(dev, needed_headroom); tailroom = BPF_CORE_READ(dev, needed_tailroom); +out: return TCX_PASS; }
diff --git a/tools/testing/selftests/bpf/progs/timer.c b/tools/testing/selftests/bpf/progs/timer.c index d6d5fef..aa966db 100644 --- a/tools/testing/selftests/bpf/progs/timer.c +++ b/tools/testing/selftests/bpf/progs/timer.c
@@ -6,6 +6,7 @@ #include <errno.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" #define CLOCK_MONOTONIC 1 #define CLOCK_BOOTTIME 7 @@ -57,9 +58,12 @@ struct { __type(key, int); __type(value, struct elem); } abs_timer SEC(".maps"), soft_timer_pinned SEC(".maps"), abs_timer_pinned SEC(".maps"), - race_array SEC(".maps"); + race_array SEC(".maps"), loop_array SEC(".maps"); __u64 bss_data; +__u64 loop_sum; +int loop_rearm; /* number of times loop_cb() re-arms itself */ +__u64 zero; __u64 abs_data; __u64 err; __u64 ok; @@ -139,6 +143,60 @@ static int timer_cb1(void *map, int *key, struct bpf_timer *timer) return 0; } +/* + * Static and not inlined, so the loop is walked in a frame below the + * callback's rather than in a separately verified global subprog. + */ +static __noinline int sum_to(__u64 n) +{ + __u64 i, sum = 0; + + for (i = zero; i < n && can_loop; i++) + sum += i; + + return sum; +} + +static int loop_cb(void *map, int *key, struct elem *val); + +/* + * Re-arms from a frame below the callback's, where the caller frame carries + * no async_entry_cnt of its own. + */ +static __noinline void rearm(struct elem *val) +{ + if (loop_rearm > 0) { + loop_rearm--; + /* set_callback is what starts another async callback entry */ + bpf_timer_set_callback(&val->t, loop_cb); + bpf_timer_start(&val->t, 0, 0); + } +} + +/* Re-arms itself and reaches a bounded loop through a call. */ +static int loop_cb(void *map, int *key, struct elem *val) +{ + loop_sum += sum_to(16); + rearm(val); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG2(test_loop_rearm, int, a) +{ + struct bpf_timer *timer; + int key = 0; + + timer = bpf_map_lookup_elem(&loop_array, &key); + if (!timer) + return 0; + + bpf_timer_init(timer, &loop_array, CLOCK_MONOTONIC); + bpf_timer_set_callback(timer, loop_cb); + bpf_timer_start(timer, 0, 0); + return 0; +} + SEC("fentry/bpf_fentry_test1") int BPF_PROG2(test1, int, a) {
diff --git a/tools/testing/selftests/bpf/progs/timer_failure.c b/tools/testing/selftests/bpf/progs/timer_failure.c index 5a2e9da..0538269 100644 --- a/tools/testing/selftests/bpf/progs/timer_failure.c +++ b/tools/testing/selftests/bpf/progs/timer_failure.c
@@ -66,3 +66,32 @@ long BPF_PROG2(test_bad_ret, int, a) return 0; } + +/* + * A real loop inside an async callback must still be rejected: both entries + * into the callback have the same async_entry_cnt, so telling entries apart + * does not apply here. + */ +static int timer_cb_infinite_loop(void *map, int *key, struct elem *val) +{ + for (;;) {} + + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__failure __msg("infinite loop detected") +long BPF_PROG2(test_infinite_loop_cb, int, a) +{ + struct bpf_timer *timer; + int key = 0; + + timer = bpf_map_lookup_elem(&timer_map, &key); + if (timer) { + bpf_timer_init(timer, &timer_map, CLOCK_BOOTTIME); + bpf_timer_set_callback(timer, timer_cb_infinite_loop); + bpf_timer_start(timer, 1000, 0); + } + + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c new file mode 100644 index 0000000..5fde710 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c
@@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__u64 base; +__u64 ret_stack; + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* g and h are on the stack, for the x86-64 kernel convention too. */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, __u64 h) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + h * 8; +} + +/* No stack args of its own, so it can share a program with a tail call. */ +static __noinline __u64 call_stack(__u64 x) +{ + return callee_stack(x + 1, x + 2, x + 3, x + 4, x + 5, x + 6, x + 7, + x + 8); +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +/* Makes the program tail_call_reachable, though callee_stack() is not. */ +static __noinline int subprog_tail(void *ctx) +{ + int ret = 0; + + bpf_tail_call_static(ctx, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = call_stack(base); +#endif + return subprog_tail(ctx); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c new file mode 100644 index 0000000..1800e7b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c
@@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +long stack_g, stack_h, stack_ret; + +SEC("fentry/callee_stack") +int stack_entry(unsigned long long *ctx) +{ + stack_g = ctx[6]; + stack_h = ctx[7]; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[8]; + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c index 4638dfe..5520ee5 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c
@@ -15,4 +15,49 @@ int test_int128_arg_fexit(unsigned long long *ctx) return 0; } +/* + * The arguments come packed in the BPF convention whatever holes the + * kernel convention left between them: an eightbyte each, and two for the + * struct holding an __int128. + */ +long s_g, s_h_lo, s_h_hi, s_i, s_ret; + +SEC("fentry/bpf_testmod_test_int128_stack") +int test_int128_stack_fentry(unsigned long long *ctx) +{ + s_g = ctx[6]; + s_h_lo = ctx[7]; + s_h_hi = ctx[8]; + s_i = ctx[9]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_stack") +int test_int128_stack_fexit(unsigned long long *ctx) +{ + s_ret = ctx[10]; + return 0; +} + +long b_f_a, b_f_b, b_g, b_h, b_i_lo, b_i_hi, b_ret; + +SEC("fentry/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fentry(unsigned long long *ctx) +{ + b_f_a = ctx[5]; + b_f_b = ctx[6]; + b_g = ctx[7]; + b_h = ctx[8]; + b_i_lo = ctx[9]; + b_i_hi = ctx[10]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fexit(unsigned long long *ctx) +{ + b_ret = ctx[11]; + return 0; +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c new file mode 100644 index 0000000..2561422 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c
@@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct int128_arg { + __int128 v; +}; + +__u64 base; +__u64 ret_odd, ret_stack, ret_split, ret_unreliable; + +#if defined(__clang__) + +const volatile bool has_struct_arg = true; + +/* arm64 would want v in x2 and x3, and b in x4. */ +__noinline __u64 callee_odd(__u64 a, struct int128_arg v, __u64 b) +{ + return a + (__u64)v.v * 2 + (__u64)((unsigned __int128)v.v >> 64) * 3 + + b * 4; +} + +#else + +const volatile bool has_struct_arg = false; + +#endif + +#if defined(__clang__) && \ + (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* + * x86-64 would want h in its third and fourth stack slots, and arm64 in + * its first and second, leaving x7 empty. + */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, + struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (__u64)h.v * 8 + (__u64)((unsigned __int128)h.v >> 64) * 9 + + i * 10; +} + +/* + * x86-64 BPF code passes h split between r9 and its first stack slot; + * arm64 passes it in x5 and x6, so nothing is split there. + */ +static __noinline __u64 callee_split(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + (__u64)h.v * 6 + + (__u64)((unsigned __int128)h.v >> 64) * 7 + i * 8; +} + +/* + * A plain pointer makes the BTF of a static function unreliable, but f is + * still passed on the stack. The store keeps p a pointer. + */ +static __noinline __u64 callee_unreliable(__u64 *p, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f) +{ + *p += 1; + return *p + b * 2 + c * 3 + d * 4 + e * 5 + f * 6; +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) + struct int128_arg v = { .v = ((__int128)(base + 9) << 64) | (base + 8) }; + + ret_odd = callee_odd(base + 1, v, base + 2); +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = callee_stack(base + 1, base + 2, base + 3, base + 4, + base + 5, base + 6, base + 7, v, base + 10); + ret_split = callee_split(base + 1, base + 2, base + 3, base + 4, + base + 5, v, base + 10); + { + __u64 x = base; + + ret_unreliable = callee_unreliable(&x, base + 2, base + 3, + base + 4, base + 5, base + 6); + } +#endif +#endif + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c new file mode 100644 index 0000000..fc81c2e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c
@@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +struct int128_arg { + __int128 v; +}; + +long odd_a, odd_lo, odd_hi, odd_b, odd_ret; + +SEC("fentry/callee_odd") +int BPF_PROG2(odd_entry, __u64, a, struct int128_arg, v, __u64, b) +{ + odd_a = a; + odd_lo = (__u64)v.v; + odd_hi = (unsigned __int128)v.v >> 64; + odd_b = b; + return 0; +} + +SEC("fexit/callee_odd") +int odd_exit(unsigned long long *ctx) +{ + odd_ret = ctx[4]; + return 0; +} + +long stack_g, stack_lo, stack_hi, stack_i, stack_ret; + +SEC("fentry/callee_stack") +int BPF_PROG2(stack_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + __u64, f, __u64, g, struct int128_arg, h, __u64, i) +{ + stack_g = g; + stack_lo = (__u64)h.v; + stack_hi = (unsigned __int128)h.v >> 64; + stack_i = i; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[10]; + return 0; +} + +long split_e, split_lo, split_hi, split_i, split_ret; + +SEC("fentry/callee_split") +int BPF_PROG2(split_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + struct int128_arg, h, __u64, i) +{ + split_e = e; + split_lo = (__u64)h.v; + split_hi = (unsigned __int128)h.v >> 64; + split_i = i; + return 0; +} + +SEC("fexit/callee_split") +int split_exit(unsigned long long *ctx) +{ + split_ret = ctx[8]; + return 0; +} + +/* Refused: the trampoline would lose the stack argument. */ +SEC("?fentry/callee_unreliable") +int unreliable_entry(unsigned long long *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/unwind_attach.c b/tools/testing/selftests/bpf/progs/unwind_attach.c new file mode 100644 index 0000000..9aff7c6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/unwind_attach.c
@@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* The targets are global_unwinds() and global_never_unwinds() in verifier_unwind.c */ + +#define unwind_exit ".8byte %[insn_unwind_exit];" +#define __imm_unwind_exit \ + __imm_insn(insn_unwind_exit, \ + BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0)) + +extern void bpf_throw(__u64 cookie) __ksym; + +int fentry_cnt; + +SEC("?fentry/global_unwinds") +int BPF_PROG(fentry_unwind) +{ + fentry_cnt++; + return 0; +} + +SEC("?fexit/global_unwinds") +int BPF_PROG(fexit_unwind) +{ + return 0; +} + +SEC("?freplace/global_unwinds") +unsigned long freplace_returns(void) +{ + return 9; +} + +SEC("?freplace/global_unwinds") +__naked unsigned long freplace_unwinds(void) +{ + asm volatile ( + "r0 = 3;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* Not naked: the argument has to have a name in BTF */ +__noinline int throws_cb(__u64 cookie) +{ + if (cookie) + asm volatile ("r0 = 3;" unwind_exit :: __imm_unwind_exit : "r0"); + return 0; +} + +/* It has no unwind_exit, so it can replace a subprog that cannot unwind */ +SEC("?freplace/global_never_unwinds") +__attribute__((btf_decl_tag("exception_callback:throws_cb"))) +unsigned long freplace_throws(void) +{ + bpf_throw(1); + return 0; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/unwind_cxx.cpp b/tools/testing/selftests/bpf/progs/unwind_cxx.cpp new file mode 100644 index 0000000..3a113ff --- /dev/null +++ b/tools/testing/selftests/bpf/progs/unwind_cxx.cpp
@@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * C++ exceptions. llvm lowers unwinding to two insns: + * invoke foo unwind pad call foo; unwind_goto pad + * resume r0 = 0; unwind_exit + * so 'throw' is a jump or an exit, 'catch (...)' and the destructors that run on + * the way are ordinary blocks. + * + * vmlinux.h and bpf_helpers.h are not C++. + */ +#define SEC(name) __attribute__((section(name), used)) +#define __ksym __attribute__((section(".ksyms"))) +#define __noinline __attribute__((noinline)) +#define __always_inline inline __attribute__((always_inline)) + +#ifdef __BPF_FEATURE_UNWIND + +struct __sk_buff { + unsigned int len; +}; + +extern "C" { +void bpf_rcu_read_lock(void) noexcept __ksym; +void bpf_rcu_read_unlock(void) noexcept __ksym; +void bpf_preempt_disable(void) noexcept __ksym; +void bpf_preempt_enable(void) noexcept __ksym; + +/* Starts an unwind. Not a kfunc: llvm lowers the call and it doesn't come back. */ +int _Unwind_RaiseException(void *exc); + +/* All that is left of the C++ runtime. Nobody looks at the exception object. */ +static char exc_obj[16]; + +__always_inline void *__cxa_allocate_exception(unsigned long size) noexcept +{ + return exc_obj; +} + +__always_inline void __cxa_throw(void *exc, void *tinfo, void (*dtor)(void *)) +{ + _Unwind_RaiseException(exc); +} + +__always_inline void *__cxa_begin_catch(void *exc) noexcept +{ + return exc; +} + +__always_inline void __cxa_end_catch(void) noexcept +{ +} +} + +/* One decimal digit per destructor, in the order they ran */ +volatile int dtors SEC(".data.dtors"); + +/* The verifier insists that both are undone on every path out of the prog */ +struct rcu_guard { + __always_inline rcu_guard() + { + bpf_rcu_read_lock(); + } + __always_inline ~rcu_guard() + { + bpf_rcu_read_unlock(); + dtors = dtors * 10 + 1; + } +}; + +struct plain_rcu_guard { + __always_inline plain_rcu_guard() + { + bpf_rcu_read_lock(); + } + __always_inline ~plain_rcu_guard() + { + bpf_rcu_read_unlock(); + } +}; + +struct preempt_guard { + __always_inline preempt_guard() + { + bpf_preempt_disable(); + } + __always_inline ~preempt_guard() + { + bpf_preempt_enable(); + dtors = dtors * 10 + 2; + } +}; + +static __noinline int inner(int len) +{ + rcu_guard guard; + + if (len > 100) + throw 1; + return len; +} + +static __noinline int middle(int len) +{ + preempt_guard guard; + + return inner(len) + 1; +} + +using u128 = unsigned __int128; + +/* Global subprogs returning 16 bytes use R0:R2, including unwind exits. */ +extern "C" __noinline u128 aggregate_return(int len) +{ + plain_rcu_guard guard; + + if (len > 100) + throw 1; + return (static_cast<u128>(len) << 64) | 7; +} + +extern "C" SEC("tc") int catches(struct __sk_buff *skb) +{ + try { + return middle(skb->len); + } catch (...) { + return 42; + } +} + +/* The exception leaves the prog: it returns 0 */ +extern "C" SEC("tc") int does_not_catch(struct __sk_buff *skb) +{ + return middle(skb->len); +} + +extern "C" SEC("tc") int catches_aggregate(struct __sk_buff *skb) +{ + try { + u128 value = aggregate_return(skb->len); + + return static_cast<unsigned long>(value) + + static_cast<unsigned long>(value >> 64); + } catch (...) { + return 42; + } +} + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c new file mode 100644 index 0000000..b0eccce --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c
@@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +struct pair { + __u64 lo; + __u64 hi; +}; + +struct small { + __u32 a; + __u32 b; +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; + __u64 hi; + } parts; +}; + +struct with_ptr { + void *p; + __u64 x; +}; + +static __noinline __u64 take_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_static_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +#if defined(__clang__) + +__noinline __u64 take_pair_global(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair_global(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_two_pairs_global(struct pair p, struct pair q, int d) +{ + return p.lo + p.hi + q.lo + q.hi + d; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_two_structs_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + struct pair q = { .lo = a + 1, .hi = b + 2 }; + + if (take_two_pairs_global(p, q, 3) != 2 * a + 2 * b + 6) + return 1; + + return 0; +} + +__noinline __u64 take_small_global(int a, struct small s, int c) +{ + return (__u64)a + s.a + s.b + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_small_struct_c_test(struct __sk_buff *skb) +{ + __u32 a = skb->len ^ (__u32)MIX_A; + __u32 b = skb->len ^ (__u32)MIX_B; + struct small s = { .a = a, .b = b }; + + if (take_small_global(1, s, 2) != (__u64)a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_upair_global(int a, union upair u, int c) +{ + return (__u64)a + u.parts.lo + u.parts.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair u; + + u.halves[0] = a; + u.halves[1] = b; + if (take_upair_global(1, u, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif + +#ifdef __SIZEOF_INT128__ + +typedef unsigned __int128 u128; + +__noinline __u64 take_i128_global(int a, u128 v, int c) +{ + return (__u64)a + (__u64)(v >> 64) + (__u64)v + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (take_i128_global(1, v, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +static __noinline __u64 take_spilled_pair(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_spilled_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + int n = skb->len; + + if (take_spilled_pair(n, n + 1, n + 2, n + 3, p) != a + b + 4 * n + 6) + return 1; + + return 0; +} + +#endif + +__noinline __u64 take_with_ptr_global(struct with_ptr s) +{ + return s.x; +} + +SEC("tc") +__failure __msg("type STRUCT in take_with_ptr_global() is not composed of scalars") +int aggregate_arg_ptr_member_fail(struct __sk_buff *skb) +{ + struct with_ptr s = { .p = skb, .x = skb->len }; + + return take_with_ptr_global(s); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c new file mode 100644 index 0000000..f5c1166 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c
@@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +typedef unsigned __int128 u128; + +struct pair { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ + } parts; +}; + +static __noinline u128 make_i128(__u64 a, __u64 b) +{ + return ((u128)(a + b) << 64) | (a - b); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = make_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +static __noinline struct pair make_pair(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +__noinline struct pair make_pair_global(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair_global(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +static __noinline union upair make_upair(__u64 a, __u64 b) +{ + union upair p; + + p.halves[0] = a + b; + p.halves[1] = a - b; + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair p; + + p = make_upair(a, b); + if (p.parts.lo != a + b) + return 1; + if (p.parts.hi != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = bpf_kfunc_call_test_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_ret_pair p; + + p = bpf_kfunc_call_test_ret_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +#else + +SEC("socket") +__description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 3e33766..1ce9c4b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c
@@ -783,4 +783,52 @@ int check_arena_arg_ret(void *ctx) return 0; } +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct arena_word_pair { + u32 __arena *first; + u32 __arena *second; +}; + +__weak struct arena_word_pair arena_word_pair(u32 __arena *page) +{ + struct arena_word_pair pair; + + pair.first = page; + pair.second = page + 1; + + return pair; +} + +SEC("syscall") +__load_if_JITed() +__success __retval(0) +int check_arena_struct_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *first, *second; + struct arena_word_pair pair; + + if (!page) + return 1; + + pair = arena_word_pair(page); + if (!pair.first || !pair.second) + return 2; + + /* Both halves of the pair must still be usable as arena pointers. */ + first = pair.first; + second = pair.second; + *first = 1; + *second = 2; + if (*first != 1) + return 3; + if (*second != 2) + return 4; + + return 0; +} + +#endif + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c index 6ab8730..dbdea14 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c
@@ -10,6 +10,9 @@ #include <bpf_arena_common.h> #define ARENA_SIZE (1ull << 32) +#define LARGE_PAGE_CNT 1025 + +volatile int zero = 0; struct { __uint(type, BPF_MAP_TYPE_ARENA); @@ -284,6 +287,7 @@ int big_alloc2(void *ctx) return 0; } +/* Nonsleepable because it binds to a socket program. */ SEC("socket") __success __retval(0) int big_alloc3(void *ctx) @@ -291,24 +295,56 @@ int big_alloc3(void *ctx) #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) char __arena *pages; u64 i; + int err = 0; - /* - * Allocate 2051 pages in one go to check how kmalloc_nolock() handles large requests. - * Since kmalloc_nolock() can allocate up to 1024 struct page * at a time, this call should - * result in three batches: two batches of 1024 pages each, followed by a final batch of 3 - * pages. - */ - pages = bpf_arena_alloc_pages(&arena, NULL, 2051, NUMA_NO_NODE, 0); + /* Verify that a nonsleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); if (!pages) - return 0; + return 1; - bpf_for(i, 0, 2051) - pages[i * PAGE_SIZE] = 123; - bpf_for(i, 0, 2051) - if (pages[i * PAGE_SIZE] != 123) - return i; + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; - bpf_arena_free_pages(&arena, pages, 2051); + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; +#endif + return 0; +} + +/* SYSCALL programs are always sleepable. */ +SEC("syscall") +__success __retval(0) +int big_alloc4(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + char __arena *pages; + u64 i; + int err = 0; + + /* Verify that a sleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); + if (!pages) + return 1; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; #endif return 0; }
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c new file mode 100644 index 0000000..6f0efd9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c
@@ -0,0 +1,1155 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, + __u64 flags) __ksym; +void bpf_arena_free_pages(void *map, void *ptr, __u32 page_cnt) __ksym; + +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START (1ull << 32) +#else +#define ARENA_VM_START (1ull << 44) +#endif + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); + __ulong(map_extra, ARENA_VM_START); +} arena SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} hash SEC(".maps"); + +/* JITs that take BPF_F_ARENA_SCALAR */ +#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 + +/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ +void __kfunc_btf_root(void) +{ + bpf_arena_alloc_pages(0, 0, 0, 0, 0); + bpf_arena_free_pages(0, 0, 0); +} + +/* Tests start with r6 = address of a new page as the user space sees it, a number */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load and store of every size") +__success __retval(0) +__load_if_JITed() +__naked void ld_st_sizes(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1122334455667788 ll; \ + *(u64 *)(r6 + 0) = r1; \ + *(u32 *)(r6 + 8) = r1; \ + *(u16 *)(r6 + 12) = r1; \ + *(u8 *)(r6 + 14) = r1; \ + *(u64 *)(r6 + 16) = 0x1234; \ + *(u32 *)(r6 + 24) = 0x5678; \ + *(u16 *)(r6 + 28) = 0x9a; \ + *(u8 *)(r6 + 30) = 0xbc; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != r1 goto 9f; \ + r0 = 2; \ + r2 = *(u32 *)(r6 + 8); \ + if r2 != 0x55667788 goto 9f; \ + r0 = 3; \ + r2 = *(u16 *)(r6 + 12); \ + if r2 != 0x7788 goto 9f; \ + r0 = 4; \ + r2 = *(u8 *)(r6 + 14); \ + if r2 != 0x88 goto 9f; \ + r0 = 5; \ + r2 = *(u64 *)(r6 + 16); \ + if r2 != 0x1234 goto 9f; \ + r0 = 6; \ + r2 = *(u32 *)(r6 + 24); \ + if r2 != 0x5678 goto 9f; \ + r0 = 7; \ + r2 = *(u16 *)(r6 + 28); \ + if r2 != 0x9a goto 9f; \ + r0 = 8; \ + r2 = *(u8 *)(r6 + 30); \ + if r2 != 0xbc goto 9f; \ + /* the address is what it was */ \ + r0 = 9; \ + if r6 != r7 goto 9f; \ + r0 = 10; \ + r7 >>= 32; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void ld_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 77; \ + *(u64 *)(r6 + 0) = r1; \ + r6 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r6 != 77 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: sign extending load") +__success __retval(0) +__load_if_JITed() +__naked void ldsx(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 0) = 0x80; \ + .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ + r0 = 1; \ + if r2 != -128 goto 9f; \ + r0 = 2; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, + BPF_REG_2, BPF_REG_6, 0, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: address loaded from arena") +__success __retval(0) +__load_if_JITed() +__naked void ptr_chase(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* page[0] = &page[64]; page[64] = 5; */ \ + r1 = r6; \ + r1 += 64; \ + *(u64 *)(r6 + 0) = r1; \ + *(u64 *)(r1 + 0) = 5; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r2 != r1 goto 9f; \ + r3 = *(u64 *)(r2 + 0); \ + r0 = 2; \ + if r3 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics") +__success __retval(0) +__load_if_JITed() +__naked void atomics(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 8) = 1; \ + r1 = 2; \ + lock *(u64 *)(r6 + 8) += r1; \ + r1 = 4; \ + .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r1 = 8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ + r0 = 2; \ + if r1 != 7 goto 9f; \ + r0 = 8; \ + r1 = 16; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != 8 goto 9f; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != 16 goto 9f; \ + r0 = 5; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching and/or/xor") +__success __retval(0) +__load_if_JITed() +__naked void fetch_and_or_xor(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = 0x11000000110 ll; \ + *(u64 *)(r6 + 8) = r7; \ + r0 = r6; \ + r0 ^= 0x55; \ + r1 = 0x1100000011 ll; \ + .8byte %[fetch_or_dw]; /* r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ + r2 = r6; \ + r2 ^= 0x55; \ + if r0 == r2 goto 1f; \ + r0 = 99; \ + goto 9f; \ +1: r0 = 1; \ + if r1 != r7 goto 9f; \ + r7 = 0x11100000111 ll; \ + r1 = 0x11000000111 ll; \ + .8byte %[fetch_and_dw]; /* r1 = atomic_fetch_and((u64 *)(r6 + 8), r1) */ \ + r0 = 2; \ + if r1 != r7 goto 9f; \ + r7 = 0x11000000111 ll; \ + r1 = 0x101 ll; \ + .8byte %[fetch_xor_dw]; /* r1 = atomic_fetch_xor((u64 *)(r6 + 8), r1) */ \ + r0 = 3; \ + if r1 != r7 goto 9f; \ + r7 = 0x11000000010 ll; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != r7 goto 9f; \ + *(u32 *)(r6 + 16) = 0x110; \ + r1 = 0xdead000000000011 ll; \ + .8byte %[fetch_or_w]; /* r1 = atomic_fetch_or((u32 *)(r6 + 16), r1) */ \ + r0 = 5; \ + if r1 != 0x110 goto 9f; \ + r1 = 0xdead000000000101 ll; \ + .8byte %[fetch_and_w]; /* r1 = atomic_fetch_and((u32 *)(r6 + 16), r1) */ \ + r0 = 6; \ + if r1 != 0x111 goto 9f; \ + r1 = 0xdead000000000011 ll; \ + .8byte %[fetch_xor_w]; /* r1 = atomic_fetch_xor((u32 *)(r6 + 16), r1) */ \ + r0 = 7; \ + if r1 != 0x101 goto 9f; \ + r2 = *(u32 *)(r6 + 16); \ + r0 = 8; \ + if r2 != 0x110 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)), + __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)), + __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching and/or/xor with the operand in r0") +__success __retval(0) +__load_if_JITed() +__naked void fetch_and_or_xor_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = 0x11000000110 ll; \ + *(u64 *)(r6 + 8) = r7; \ + r0 = 0x1100000011 ll; \ + .8byte %[fetch_or_dw]; /* r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != r7 goto 9f; \ + r7 = 0x11100000111 ll; \ + r0 = 0x11000000111 ll; \ + .8byte %[fetch_and_dw]; /* r0 = atomic_fetch_and((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 2; \ + if r2 != r7 goto 9f; \ + r7 = 0x11000000111 ll; \ + r0 = 0x101 ll; \ + .8byte %[fetch_xor_dw]; /* r0 = atomic_fetch_xor((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != r7 goto 9f; \ + r7 = 0x11000000010 ll; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != r7 goto 9f; \ + *(u32 *)(r6 + 16) = 0x110; \ + r0 = 0xdead000000000011 ll; \ + .8byte %[fetch_or_w]; /* r0 = atomic_fetch_or((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 5; \ + if r2 != 0x110 goto 9f; \ + r0 = 0xdead000000000101 ll; \ + .8byte %[fetch_and_w]; /* r0 = atomic_fetch_and((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 6; \ + if r2 != 0x111 goto 9f; \ + r0 = 0xdead000000000011 ll; \ + .8byte %[fetch_xor_w]; /* r0 = atomic_fetch_xor((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 7; \ + if r2 != 0x101 goto 9f; \ + r2 = *(u32 *)(r6 + 16); \ + r0 = 8; \ + if r2 != 0x110 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)), + __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)), + __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)) + : __clobber_all); +} + +/* + * The page is gone, so the access faults. The fetched register has to come back + * 0 and everything else has to be left alone, r0 included. Reading the address + * again has to give 0 as well: had the fault been recovered by the arena mapping + * its zeroed scratch page instead of by the JIT's exception table entry, the + * access would have succeeded and left a 1 there. + */ +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching or through a number to a freed page") +__success __retval(0) +__load_if_JITed() +__naked void fetch_or_freed_page(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x55; \ + r1 = %[arena] ll; \ + r2 = r6; \ + r3 = 1; \ + call %[bpf_arena_free_pages]; \ + r0 = r6; \ + r0 ^= 0x55; \ + r1 = 1; \ + .8byte %[fetch_or_dw]; /* faults: r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ + r2 = r6; \ + r2 ^= 0x55; \ + if r0 == r2 goto 1f; \ + r0 = 1; \ + goto 9f; \ +1: r0 = 2; \ + if r1 != 0 goto 9f; \ + r3 = *(u64 *)(r6 + 8); \ + r0 = 3; \ + if r3 != 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_arena_free_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching or with the operand in r0 to a freed page") +__success __retval(0) +__load_if_JITed() +__naked void fetch_or_r0_freed_page(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x55; \ + r1 = %[arena] ll; \ + r2 = r6; \ + r3 = 1; \ + call %[bpf_arena_free_pages]; \ + r0 = 1; \ + .8byte %[fetch_or_dw]; /* faults: r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 0 goto 9f; \ + r3 = *(u64 *)(r6 + 8); \ + r0 = 2; \ + if r3 != 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_arena_free_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: cmpxchg through r0") +__success __retval(0) +__load_if_JITed() +__naked void cmpxchg_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* The address is in r0. The value at the address is not equal to it. */ \ + *(u64 *)(r6 + 0) = 3; \ + r0 = r6; \ + r1 = 5; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void xchg_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 0) = 3; \ + r1 = r6; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != r6 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds a pointer to stack") +__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") +__naked void xchg_into_stack_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r1 = r10; \ + r1 += -8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load-acquire and store-release") +__success __retval(0) +__load_if_JITed() +__naked void load_acq_store_rel(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1234; \ + .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ + .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ + r0 = 1; \ + if r2 != 0x1234 goto 9f; \ + .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ + r0 = 2; \ + if r2 != 0x34 goto 9f; \ + r0 = 3; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), + __imm_insn(load_acquire8_insn, + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void not_in_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + /* nothing is allocated: all loads read 0, stores are dropped */ \ + r6 = 0xdeadbeef00000000 ll; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != 0 goto 9f; \ + r0 = 2; \ + r2 = *(u8 *)(r6 - 32768); \ + if r2 != 0 goto 9f; \ + r6 = 0x12345678ffffffff ll; \ + r0 = 3; \ + r2 = *(u64 *)(r6 + 32760); \ + if r2 != 0 goto 9f; \ + *(u64 *)(r6 + 32760) = 1; \ + *(u8 *)(r6 + 32767) = r2; \ + r1 = 1; \ + lock *(u64 *)(r6 + 32760) += r1; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through the address of the stack as a number") +__success __retval(0) +__load_if_JITed() +__naked void stack_addr_as_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + *(u64 *)(r10 - 8) = 5; \ + r6 = r10; \ + r6 |= 0; \ + /* r6 is a number now. The store goes to arena, not to the stack. */ \ + *(u64 *)(r6 - 8) = 7; \ + r1 = 9; \ + *(u64 *)(r6 - 8) = r1; \ + lock *(u64 *)(r6 - 8) += r1; \ + r2 = *(u64 *)(r10 - 8); \ + r0 = 1; \ + if r2 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: 64-bit math on the address stays 64-bit") +__success __retval(0) +__load_if_JITed() +__naked void alu64_after_access(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r7 = r6; \ + r7 += 8; \ + r7 -= r6; \ + r0 = 1; \ + if r7 != 8 goto 9f; \ + r7 = r6; \ + r7 >>= 32; \ + r0 = 2; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: st of an immediate changes no register") +__success __retval(0) +__load_if_JITed() +__naked void st_keeps_regs(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r0 = r6; \ + r1 = 0x1111; \ + r2 = 0x2222; \ + *(u64 *)(r0 + 0) = 5; \ + r3 = r0; \ + r0 = 1; \ + if r3 != r6 goto 9f; \ + r0 = 2; \ + if r1 != 0x1111 goto 9f; \ + r0 = 3; \ + if r2 != 0x2222 goto 9f; \ + r0 = 0x3333; \ + r1 = r6; \ + *(u32 *)(r1 + 8) = -7; \ + r3 = r0; \ + r0 = 4; \ + if r3 != 0x3333 goto 9f; \ + r0 = 5; \ + if r1 != r6 goto 9f; \ + r0 = 6; \ + r3 = *(u64 *)(r6 + 0); \ + if r3 != 5 goto 9f; \ + r0 = 7; \ + r3 = *(u64 *)(r6 + 8); \ + r4 = 0xfffffff9 ll; \ + if r3 != r4 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number with garbage in the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_garbage(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number without the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_small(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0x1000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics through a number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void atomic_value(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + lock *(u64 *)(r8 + 0) += r1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + .8byte %[fetch_add_insn]; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + .8byte %[xchg_insn]; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ + .8byte %[cmpxchg_insn]; \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to arena at the same insn") +__success __retval(0) +__load_if_JITed() +__naked void mixed_number_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x1234; \ + /* a number that the verifier does not know, 0 at run time */ \ + r7 = *(u64 *)(r6 + 16); \ + r8 = r6; \ + if r7 != 0 goto 1f; \ + .8byte %[cast_kern_insn]; \ +1: *(u8 *)(r8 + 0) = 1; \ + *(u8 *)(r8 + 1) = r7; \ + r2 = *(u8 *)(r8 + 1); \ + lock *(u64 *)(r8 + 24) += r7; \ + r0 = 0; \ + if r7 != 0 goto 9f; \ + /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ + r0 = 2; \ + r2 = *(u64 *)(r8 + 8); \ + if r2 != 0x1234 goto 9f; \ + r0 = 3; \ + r2 = *(u8 *)(r6 + 0); \ + if r2 != 1 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, + BPF_REG_8, BPF_REG_8, 1, 1)) + : __clobber_all); +} + +SEC("syscall") +__description("arena_scalar: no flag, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_flag(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + /* nothing is allocated: the load reads 0 */ \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL with an offset is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr_off(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + r1 += 8; \ + *(u64 *)(r1 + 0) = 5; \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is less than a page is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void small_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r1 &= 0xfff; \ + r0 = *(u64 *)(r1 + 0); \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number is not a pointer for a helper") +__failure __msg("R2 type=scalar expected=") +__naked void helper_arg(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = %[hash] ll; \ + r2 = r6; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_arena_alloc_pages), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to stack at the same insn") +__failure __msg("same insn cannot be used with different pointers") +__load_if_JITed() +__naked void mixed_number_stack(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r10 - 8) = 0; \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r6 = r10; \ + r6 += -8; \ +1: r0 = *(u64 *)(r6 + 0); \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static int st_cb(__u64 idx, void *ctx) +{ + volatile long *p = *(volatile long **)ctx; + + p[idx] = 7; + return 0; +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through a number in a callback") +__success __retval(0) +__load_if_JITed() +int st_in_callback(void *unused) +{ + volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); + + if (!p) + return 100; + bpf_loop(4, st_cb, &p, 0); + return p[0] + p[3] + p[4] - 14; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c new file mode 100644 index 0000000..d824a5d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
@@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* Tests for the linear "base + step * k" description tracked per scalar register. */ + +#define __no_step __not_msg("step=") __msg("\n") + +/* + * Safe ALU64 arithmetic preserves a non-power-of-two stride and scales its base: + * step=0+3 -> step=1+3 -> step=2+6 -> step=4+12. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 *= 3 {{.*}}step=0+3)") +__msg("r0 += 1 {{.*}}step=1+3)") +__msg("r0 *= 2 {{.*}}step=2+6)") +__msg("r0 <<= 1 {{.*}}step=4+12)") +__naked void step_arith64_no_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + r0 *= 3; \ + r0 += 1; \ + r0 *= 2; \ + r0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Safe ALU32 multiplication and addition preserve the stride and scale its base: + * step=0+3 -> step=1+3 -> step=2+6. MOV32 preserves the result. + * LSH32 currently resets the stride, but must preserve the correct upper bound. + */ +SEC("socket") +__success __log_level(2) +__msg("w0 *= 3 {{.*}}step=0+3)") +__msg("w0 += 1 {{.*}}step=1+3)") +__msg("w0 *= 2 {{.*}}step=2+6)") +__msg("w1 = w0 {{.*}}R0=scalar({{.*}}step=2+6) R1=scalar({{.*}}step=2+6)") +__msg("w0 <<= 1 {{.*}}smax=umax=smax32=umax32=3064,") __no_step +__naked void step_arith32_no_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + w0 *= 3; \ + w0 += 1; \ + w0 *= 2; \ + w1 = w0; \ + w0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Sign-crossing range with a non-power-of-2 step. After "*= 3; += -3" the value + * set is {-3, 0, 3, 6}. The line description is tracked in signed space, so the + * intersection keeps smax=6. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += -3 {{.*}}smax=smax32=6,step=0+3)") +__naked void step_neg_value_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r0 *= 3; \ + r0 += -3; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * 0x49249249 * 7 wraps to U32_MAX; inferring step=0+7 would incorrectly + * cap the result at 0xfffffffc instead of U32_MAX. + * Adding 4 to {0xfffffffc, U32_MAX} wraps to {0, 3}; keeping step=1+3 + * incorrectly narrows the result to 1. + * Shifting the same input left by 1 gives {0xfffffff8, 0xfffffffe}; + * inferring step=0+6 incorrectly narrows the result to 0xfffffffc. + */ +SEC("socket") +__success __log_level(2) +__msg("w0 *= 7 {{.*}}umax=0xffffffff,") __no_step +__msg("r2 = r0 {{.*}}R0=scalar({{.*}}step=0+3) R2=scalar({{.*}}step=0+3)") +__msg("w0 += 4 {{.*}}smax=umax=smax32=umax32=3,") __no_step +__msg("w2 <<= 1 {{.*}}smax=umax=umax32=0xfffffffe,") __no_step +__naked void step_arith32_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w0 *= 7; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r1 = 0xfffffffc ll; \ + r0 += r1; \ + r2 = r0; \ + w0 += 4; \ + w2 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Multiplying {0x2aaaaaaaaaaaaaab, 0x2aaaaaaaaaaaaaac} by 3 crosses S64_MAX + * and gives {S64_MIN + 1, S64_MIN + 4}. Keeping step=0+3 excludes both. + * + * Adding 5 to {S64_MAX - 4, S64_MAX - 1} gives {S64_MIN, S64_MIN + 3}. + * Updating the line to step=2+3 incorrectly narrows it to S64_MIN + 1. + * + * Adding -5 to {S64_MIN + 1, S64_MIN + 4} wraps to + * {S64_MAX - 3, S64_MAX}. The stale base 0 modulo 3 excludes S64_MAX. + * + * Shifting {0x4000000000000008, 0x400000000000000b} left by 1 gives + * {S64_MIN + 16, S64_MIN + 22}. Keeping step=0+6 excludes both. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 *= 3 {{.*}}smax=0x8000000000000004,") __no_step +__msg("r0 += r1 {{.*}}step=0+3)") +__msg("r0 += 5 {{.*}}smax=0x8000000000000003,") __no_step +__msg("r0 += r1 {{.*}}step=2+3)") +__msg("r0 += -5 {{.*}}umax=0x7fffffffffffffff,") __no_step +__msg("r0 += r1 {{.*}}step=0+3)") +__msg("r0 <<= 1 {{.*}}smax=0x8000000000000016,") __no_step +__naked void step_arith64_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r2 = r0; \ + r1 = 0x2aaaaaaaaaaaaaab ll; \ + r0 += r1; \ + r0 *= 3; \ + r2 *= 3; \ + r0 = r2; \ + r1 = 0x7ffffffffffffffb ll; \ + r0 += r1; \ + r0 += 5; \ + r0 = r2; \ + r1 = 0x8000000000000001 ll; \ + r0 += r1; \ + r0 += -5; \ + r0 = r2; \ + r1 = 0x4000000000000008 ll; \ + r0 += r1; \ + r0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * MOV32 maps {2^32, 2^32 + 3} to {0, 3}. The old base 1 modulo 3 + * excludes both results. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += r1 {{.*}}step=1+3)") +__msg("w0 = w0 {{.*}}smax=umax=smax32=umax32=3,") __no_step +__naked void step_mov32_truncate(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r1 = 0x100000000 ll; \ + r0 += r1; \ + w0 = w0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * MOV32 maps {-4, -1} to {0xfffffffc, U32_MAX}. The old base 2 + * modulo 3 excludes U32_MAX. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += -4 {{.*}}step=2+3)") +__msg("w0 = w0 {{.*}}umax=0xffffffff,") __no_step +__naked void step_mov32_negative(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r0 += -4; \ + w0 = w0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Sign-extending {128, 131} from s8 gives {-128, -125}. + * Keeping base 2 modulo 3 incorrectly narrows the result to -127. + * + * Sign-extending {32768, 32771} from s16 gives {-32768, -32765}. + * Keeping base 2 modulo 3 incorrectly narrows the result to -32767. + * + * Sign-extending {2^31, 2^31 + 3} from s32 gives {S32_MIN, S32_MIN + 3}. + * Keeping base 2 modulo 3 incorrectly narrows the result to S32_MIN + 1. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += 128 {{.*}}step=2+3)") +__msg("r0 = (s8)r0 {{.*}}smax=smax32=-125,") __no_step +__msg("r0 += 32768 {{.*}}step=2+3)") +__msg("r0 = (s16)r0 {{.*}}smax=smax32=-32765,") __no_step +__msg("r0 += r1 {{.*}}step=2+3)") +__msg("r0 = (s32)r0 {{.*}}smax=0xffffffff80000003,") __no_step +__naked void step_movsx(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r2 = r0; \ + r0 += 128; \ + r0 = (s8)r0; \ + r0 = r2; \ + r0 += 32768; \ + r0 = (s16)r0; \ + r0 = r2; \ + r1 = 0x80000000 ll; \ + r0 += r1; \ + r0 = (s32)r0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * R1 = R0 + 1 shares R0's scalar ID. Refining R0 to {0, 3, 6} must + * translate its line as well as its bounds when updating R1 to {1, 4, 7}. + * Copying R0's base 0 modulo 3 excludes the reachable value 7. + */ +SEC("socket") +__success __log_level(2) +__msg("r1 += 1 {{.*}}step=1+3)") +__msg("if r0 > 0x6 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=7,{{.*}}step=1+3)") +__naked void step_linked_regs_add64(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r0 *= 3; \ + r1 = r0; \ + r1 += 1; \ + if r0 > 6 goto l_out_%=; \ + r0 = r1; \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r0 &= 7 -> [0, 7] + * r0 *= 24 -> [0, 168] step=0+24 + * r0 += 8 -> [8, 176] step=8+24 + * r0 & 0x10 -> [32, 176] step=8+24 (var_off alone: umin=16) + * r0 > 80 -> [104, 176] step=8+24 (var_off alone: umin=88) + * r1 += 24 -> [128, 200] step=8+24 + * Bounds and step stay in sync after each refinement. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += 8 {{.*}}smin=umin=smin32=umin32=8,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("from 4 to 6: R0=scalar(smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 = r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=umin=smin32=umin32=104,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 += 24 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=128,smax=umax=smax32=umax32=200,{{.*}}step=8+24)") +__naked void step_bounds_resync_after_arith(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 7; \ + r0 *= 24; \ + r0 += 8; \ + if r0 & 0x10 goto l_set_%=; \ + goto l_out_%=; \ +l_set_%=: \ + if r0 <= 80 goto l_out_%=; \ + r1 = r0; \ + r1 += 24; \ + *(u64 *)(r10 - 8) = r1; \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r0 &= 7 -> [0, 7] + * r0 *= 24 -> [0, 168] step=0+24 + * r1 = (s16)r0 -> [0, 168] step=0+24 + * r0 += -24 -> [-24, 144] step=0+24 + * r2 = (s16)r0 -> [-24, 144] step=0+24 + * r3 = (s8)r0 -> [-128, 127] no step + * w4 = (s16)w1 -> [0, 168] step=0+24 + * w5 = (s8)w1 -> [-128, 127] no step + * Bounds and step stay in sync, the step is dropped when the value does not fit. + */ +SEC("socket") +__success __log_level(2) +__msg("r1 = (s16)r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("r2 = (s16)r0 {{.*}}R2=scalar(smin=smin32=-24,smax=smax32=144,{{.*}}step=0+24)") +__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)") +__msg("w4 = (s16)w1 {{.*}}R4=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,var_off=(0x0; 0xffffffff))") +__naked void step_movsx_bounds_on_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 7; \ + r0 *= 24; \ + r1 = (s16)r0; \ + r0 += -24; \ + r2 = (s16)r0; \ + r3 = (s8)r0; \ + w4 = (s16)w1; \ + w5 = (s8)w1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +struct step_val { + __u8 data[1024]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct step_val); +} step_map SEC(".maps"); + +/* Old register [4..130, step 2] should prune cur register [8..68, step 4]. */ +SEC("socket") +__success __log_level(2) +__msg("7: (27) r6 *= 4 ; R6=scalar({{.*}}umin32=8,{{.*}}umax32=68,{{.*}},step=0+4)") +__msg("10: (27) r7 *= 2 ; R7=scalar({{.*}}umin32=4,{{.*}}umax32=130,{{.*}},step=0+2)") +__msg("11: (25) if r0 > 0x2a goto pc+1") +__msg("from 11 to 13: safe") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void step_prune_hit_multiple(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + call %[bpf_get_prandom_u32]; \ + r6 &= 0x0f; \ + r6 += 2; \ + r6 *= 4; \ + r7 &= 0x3f; \ + r7 += 2; \ + r7 *= 2; \ + if r0 > 42 goto 1f; /* can't predict */ \ + r6 = r7; /* step=2 explored first, step=4 explored next */ \ +1: r0 = r10; \ + r6 = -r6; \ + r0 += r6; \ + *(u8 *)(r0 + 0) = 7; /* force r6 precise */ \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Old register [0..189, step 3] should not prune cur register [0..30, step 2]. + * Bounds and tnum containment allow pruning; step divisibility must prevent it. + */ +SEC("socket") +__success __log_level(2) +__msg("6: (27) r6 *= 2 ; R6=scalar({{.*}}smin32=0,{{.*}}umax32=30,{{.*}},step=0+2)") +__msg("8: (27) r7 *= 3 ; R7=scalar({{.*}}smin32=0,{{.*}}umax32=189,{{.*}},step=0+3)") +__msg("9: (25) if r0 > 0x2a goto pc+1") +__msg("17: (95) exit") +__msg("from 9 to 11: R6=scalar({{.*}},step=0+2)") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void step_prune_miss_non_multiple(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + call %[bpf_get_prandom_u32]; \ + r6 &= 0x0f; \ + r6 *= 2; \ + r7 &= 0x3f; \ + r7 *= 3; \ + if r0 > 42 goto 1f; /* can't predict */ \ + r6 = r7; /* step=3 explored first, step=2 explored next */ \ +1: \ + r2 = r10; \ + r6 += 1; \ + r6 = -r6; \ + r2 += r6; \ + *(u8 *)(r2 + 0) = 7; /* force r6 precise */ \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Constant current register lying on the cached line: cached is {0,3,6,...} + * (step 3, base 0), current is the constant 6. range_within() takes the + * constant branch: imod(6, 3) == base 0, so cur is on the line and the + * (precise) state is pruned -> "safe". + */ +SEC("socket") +__success __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("14: (27) r1 *= 3") /* cached path: step 3 line */ +__msg("16: (b7) r1 = 6") /* current path: const 6, on the line */ +__msg("17: safe") /* pruned at join: imod(6, 3) == 0 */ +__naked void step_prune_hit_const_on_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[step_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l_out_%=; \ + r7 = r0; \ + r1 = r6; \ + r1 &= 0xff; \ + if r6 > 0 goto l_cur_%=; \ + r1 *= 3; /* old: step 3 */ \ + goto l_join_%=; \ +l_cur_%=: \ + r1 = 6; /* cur: const on line */ \ +l_join_%=: \ + r0 = r7; \ + r0 += r1; /* r1 forced precise */ \ + r2 = *(u8 *)(r0 + 0); \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(step_map) + : __clobber_all); +} + +/* + * Constant current register NOT on the cached line: cached is {0,3,6,...} + * (step 3, base 0), current is the constant 7. imod(7, 3) == 1 != base 0, + * so range_within() fails and the join is traversed again. A power-of-two + * step is avoided on purpose: with step 3 the tnum is loose enough to admit + * 7, so imod() is + * the sole check that rejects it. No failure shape is possible here: 7 is + * within the line's bounds and tnum, and the cached line path already + * verifies the whole outro, so an eager prune could not miss an error. + */ +SEC("socket") +__success __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("14: (27) r1 *= 3") /* cached path: step 3 line */ +__msg("16: (b7) r1 = 7") /* current path: const 7, off the line */ +/* not pruned: current continues past the join with the constant offset 7 */ +__msg("19: R0=map_value({{.*}}imm=7) R1=7") +__naked void step_prune_miss_const_off_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[step_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l_out_%=; \ + r7 = r0; \ + r1 = r6; \ + r1 &= 0xff; \ + if r6 > 0 goto l_cur_%=; \ + r1 *= 3; /* old: step 3 */ \ + goto l_join_%=; \ +l_cur_%=: \ + r1 = 7; /* cur: const off line */ \ +l_join_%=: \ + r0 = r7; \ + r0 += r1; /* r1 forced precise */ \ + r2 = *(u8 *)(r0 + 0); \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(step_map) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index a73b837..e0c3891 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
@@ -502,6 +502,175 @@ __naked void bad_helper_write(void) : __clobber_all); } +/* + * A helper buffer or a callee's pointer that reaches a fastcall spill slot + * must keep the spill/fill pair and the stack that covers it. Only load + * these programs: without the fix, they access kernel stack outside their + * frame. + * + * Uninitialized outputs without CAP_PERFMON have no explicit stack accesses. + */ +SEC("tc") +__log_level(4) +__msg_unpriv("subprog 0 (helper_uninit_stack) main {{.*}} stack 64") +__xlated_unpriv("*(u64 *)(r10 -8) = r1") +__xlated_unpriv("...") +__xlated_unpriv("r1 = *(u64 *)(r10 -8)") +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void helper_uninit_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r2 = 0;" + "r3 = r10;" + "r3 += -64;" + "r4 = 8;" + "call %[bpf_skb_load_bytes];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +/* Same output in the caller's stack, passed to the helper by a callee. */ +static __used __naked void helper_uninit_stack_callee(void) +{ + asm volatile ( + "r3 = r2;" + "r2 = 0;" + "r4 = 8;" + "call %[bpf_skb_load_bytes];" + "exit;" + : + : __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +SEC("tc") +__log_level(4) +__msg_unpriv("subprog 0 (helper_uninit_stack_caller) main {{.*}} stack 64") +__xlated_unpriv("*(u64 *)(r10 -8) = r1") +__xlated_unpriv("...") +__xlated_unpriv("r1 = *(u64 *)(r10 -8)") +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void helper_uninit_stack_caller(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r2 = r10;" + "r2 += -64;" + "call helper_uninit_stack_callee;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A helper input whose only initialization is the fastcall spill. */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} fastcall_map SEC(".maps"); + +SEC("tc") +__log_level(4) +__msg("subprog 0 (helper_reads_fastcall_spill) main {{.*}} stack 8") +__xlated("*(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("r1 = *(u64 *)(r10 -8)") +__success +__naked void helper_reads_fastcall_spill(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = %[fastcall_map] ll;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_map_lookup_elem];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_map_lookup_elem), + __imm_addr(fastcall_map) + : __clobber_all); +} + +/* A callee load from the caller's fastcall spill slot. */ +static __used __naked void read_caller_stack_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("raw_tp") +__log_level(4) +__msg("subprog 0 (callee_reads_fastcall_spill) main {{.*}} stack 8") +__xlated("*(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("r1 = *(u64 *)(r10 -8)") +__success +__naked void callee_reads_fastcall_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call read_caller_stack_callee;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A zero-sized buffer touches no stack, the rewrite is still applied. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (helper_zero_size_buffer) main {{.*}} stack 0") +__xlated("0: r1 = 1") +__xlated("1: r0 =") +__success +__naked void helper_zero_size_buffer(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -64;" + "r2 = 0;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_probe_read_kernel) + : __clobber_all); +} + SEC("raw_tp") __arch_x86_64 /* main, not patched */
diff --git a/tools/testing/selftests/bpf/progs/verifier_bswap.c b/tools/testing/selftests/bpf/progs/verifier_bswap.c index cffaf36..48ffb5b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bswap.c +++ b/tools/testing/selftests/bpf/progs/verifier_bswap.c
@@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_BSWAP)) SEC("socket") __description("BSWAP, 16") @@ -117,6 +117,7 @@ l0_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c new file mode 100644 index 0000000..9dc2c9a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c
@@ -0,0 +1,577 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> +#include "bpf_experimental.h" +#include "bpf_misc.h" + +/* + * Variable-offset access into arrays within a BTF access chain, e.g. a[i].b, + * where 'i' is a register with a varying offset. The register's step has to + * match the stride of one of the arrays crossed while walking the access + * chain, and the maximal possible offset has to stay within that array. + * + * A program-allocated object (bpf_obj_new) is used as the source of a + * PTR_TO_BTF_ID with a shape we fully control. + */ +struct inner { + int a; + int b; +}; + +struct outer { + struct inner arr[8]; /* off 0, stride 8, size 64 */ + char bytes[32]; /* off 64, stride 1, size 32 */ + long longs[8]; /* off 96, stride 8, size 64 */ + int grid[4][4]; /* off 160, 16 ints, size 64 */ +}; + +/* arr[i].b, i in [0, 7], 4-byte read: stays within arr. */ +SEC("syscall") +__success +int arr_field_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* arr[i].b, i in [0, 15]: max offset runs past the end of arr. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int arr_field_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* longs[i], i in [0, 7], 8-byte read: scalar array with stride 8. */ +SEC("syscall") +__success +int long_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 96); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * bytes[i], i in [0, 31], read as 4 bytes: min_off is valid within the array, + * but the 4-byte read at max_off runs past the array end. Exercises the + * size-aware bound in btf_struct_access(). + */ +SEC("syscall") +__failure +__flag(BPF_F_ANY_ALIGNMENT) +__msg("invalid variable offset access into struct outer") +int byte_array_size_spanning(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 64); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* bytes[i], i in [0, 31], 1-byte read: step 1 matches the char array stride. */ +SEC("syscall") +__success +int byte_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 64); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * arr[] accessed with step 4 (half of sizeof(struct inner)): the step is not + * a whole number of elements, so no crossed array has a matching stride. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int arr_unaligned_step(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * grid[i][0], i in [0, 3]: steps along the outer dimension of a 2D array + * (stride 16). __btf_resolve_size() linearizes the array; the step is a + * multiple of the innermost element size (4). + */ +SEC("syscall") +__success +int grid_outer_dim(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= 16; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 160); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * grid[0][i], i in [0, 3]: steps along the inner dimension of a 2D array + * (stride 4, the innermost element size). + */ +SEC("syscall") +__success +int grid_inner_dim(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 160); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * An array of pointers within a program-allocated object. Reading a pointer + * member of a local object yields a SCALAR_VALUE, but the varying offset still + * has to be validated against the array bounds. + */ +struct with_ptrs { + struct inner *parr[8]; /* off 0, stride 8 (sizeof ptr), size 64 */ +}; + +/* parr[i], i in [0, 7], 8-byte read: stays within parr. */ +SEC("syscall") +__success +int ptr_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct with_ptrs *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * parr[i], i in [0, 15]: max offset runs past the end of parr. Without the + * variable-offset check on the WALK_PTR path this would be wrongly accepted. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct with_ptrs") +int ptr_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct with_ptrs *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * The cases above all use bpf_obj_new() (PTR_TO_BTF_ID | MEM_ALLOC). Exercise + * the trusted kernel-BTF path too, using a task_struct from + * bpf_get_current_task_btf() and its embedded char comm[] array. The comm + * offset is folded into the pointer as a constant before the varying index. + */ + +/* task->comm[i], i in [0, 15], 1-byte read: stays within comm. */ +SEC("syscall") +__success +int kernel_btf_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct task_struct *task; + int val = 0; + + task = bpf_get_current_task_btf(); + asm volatile (" \ + r1 = %[task]; \ + r1 += %[off]; \ + r2 = %[i]; \ + r2 &= 15; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 0); \ +" : [val] "=r"(val) + : [task] "r"(task), [i] "r"(i), + [off] "i"(offsetof(struct task_struct, comm)) + : "r1", "r2"); + return val; +} + +/* task->comm[i], i in [0, 31]: max offset runs past comm. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct task_struct") +int kernel_btf_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct task_struct *task; + int val = 0; + + task = bpf_get_current_task_btf(); + asm volatile (" \ + r1 = %[task]; \ + r1 += %[off]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 0); \ +" : [val] "=r"(val) + : [task] "r"(task), [i] "r"(i), + [off] "i"(offsetof(struct task_struct, comm)) + : "r1", "r2"); + return val; +} + +/* + * Array nested below a struct member at a non-zero offset. The walk descends + * through 'm' (resetting the running offset) before reaching 'arr', exercising + * the array_start = min_off - arrays[i].off telescoping across a WALK_STRUCT + * dive. + */ +struct mid { + struct inner arr[8]; /* stride 8, size 64 */ +}; + +struct nest { + long pad; /* off 0 */ + struct mid m; /* off 8 */ +}; + +/* m.arr[i].b, i in [0, 7], 4-byte read: stays within m.arr. */ +SEC("syscall") +__success +int nested_struct_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct nest *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 12); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* m.arr[i].b, i in [0, 15]: max offset runs past the end of m.arr. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct nest") +int nested_struct_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct nest *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 12); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * A trailing flexible array member makes the struct tail an unbounded region, + * so a varying access into it has no upper bound to exceed (mirrors how + * unix_address.name[] is accessed via sun_path[i]). + * + * A custom (program) BTF struct can only be reached as a PTR_TO_BTF_ID through + * bpf_obj_new(), which does not allow flexible array members. So use a kernel + * type via an untrusted PTR_TO_BTF_ID (bpf_core_cast()). struct vring_used ends + * with a flexible array 'ring[]' of struct vring_used_elem { __virtio32 id, len; }, + * mirroring a 'struct foo { int a; int b; }' flexible array. + */ + +/* ring[i].len, i in [0, 63], 4-byte read: the flexible array has no upper bound. */ +SEC("syscall") +__success +int flex_array_field(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct vring_used *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct vring_used); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 63; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 8); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + return val; +} + +/* + * ring[] accessed with step 4 (half of sizeof(struct vring_used_elem)): the + * step is not a whole number of elements, so the flexible array stride does + * not match. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct vring_used") +int flex_array_unaligned_step(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct vring_used *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct vring_used); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + return val; +} + +/* + * Access either name[0].sun_path[0] or one byte past sun_path. + * Step 108 does not match sizeof(struct sockaddr_un), which is 110. + * It matches the inner char array's stride, but exceeds its bounds. + * The flexible-tail exemption must not apply to that fixed array. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct unix_address") +int flex_array_inner_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct unix_address *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct unix_address); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 1; \ + r2 *= %[len]; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + %[off]); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i), + __imm_const(len, sizeof_field(struct sockaddr_un, sun_path)), + __imm_const(off, offsetof(struct unix_address, name) + + offsetof(struct sockaddr_un, sun_path)) + : "r1", "r2"); + return val; +} + +typedef int array_element_t; +typedef array_element_t array_type_t[4]; + +struct typedef_array { + array_type_t values; +}; + +/* + * Resolving values' size crosses both the array and element typedefs. + * Record the array ID, not the int ID left by element-type resolution. + * values[i], i in [0, 3], must pass the variable-offset array check. + */ +SEC("syscall") +__success +int typedef_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct typedef_array *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= %[elem_size]; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i), + __imm_const(elem_size, sizeof(array_element_t)) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c new file mode 100644 index 0000000..7e59188 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c
@@ -0,0 +1,1086 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx: indirect calls of bpf subprogs */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* gcc doesn't support indirect calls */ +#if defined(__clang__) && (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + +#define CALLX_INSN(DST, SRC, OFF, IMM) \ + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, DST, SRC, OFF, IMM) + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} map_array SEC(".maps"); + +struct val_with_lock { + struct bpf_spin_lock lock; + int cnt; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct val_with_lock); +} map_lock SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} map_prog SEC(".maps"); + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* apply(fn, x) { return fn(x); } */ +__naked __noinline __used +static unsigned long apply(void) +{ + asm volatile ( + "r3 = r1;" + "r1 = r2;" + "callx r3;" + "exit;" + ); +} + +SEC("socket") +__success __retval(6) +__naked void callx_basic(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* callx is printed by the verifier log and xlated dump */ +SEC("socket") +__success __log_level(2) +__msg("(8d) callx r2") +__xlated("callx r2") +__naked void callx_disasm(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* different callees are called by the same callx on different paths */ +SEC("socket") +__success __retval(11) +__naked void callx_two_callees(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r2 = %[add1] ll;" + "if r6 == 0 goto +2;" + "r2 = %[add2] ll;" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +/* pointer to a function is passed as an argument */ +SEC("socket") +__success __retval(45) +__naked void callx_fn_as_arg(void) +{ + asm volatile ( + "r1 = %[add2] ll;" + "r2 = 40;" + "call apply;" + "r6 = r0;" + "r1 = %[add1] ll;" + "r2 = 2;" + "call apply;" + "r0 += r6;" + "exit;" + : + : __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long get_add1(void) +{ + asm volatile ( + "r0 = %[add1] ll;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* pointer to a function is returned from a subprog and called via r0 */ +SEC("socket") +__success __retval(2) +__naked void callx_r0(void) +{ + asm volatile ( + "call get_add1;" + "r1 = 1;" + "callx r0;" + "exit;" + ::: __clobber_all); +} + +/* pointer to a function survives spill/fill */ +SEC("socket") +__success __retval(9) +__naked void callx_spill_fill(void) +{ + asm volatile ( + "r2 = %[add2] ll;" + "*(u64 *)(r10 - 8) = r2;" + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "r9 = *(u64 *)(r10 - 8);" + "callx r9;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long clobber_callee_saved(void) +{ + asm volatile ( + "r6 = 100;" + "r7 = 100;" + "r8 = 100;" + "r9 = 100;" + "r0 = r1;" + "exit;" + ); +} + +/* r6-r9 are preserved across callx */ +SEC("socket") +__success __retval(11) +__naked void callx_callee_saved_regs(void) +{ + asm volatile ( + "r6 = 1;" + "r7 = 2;" + "r8 = 3;" + "r9 = 4;" + "r1 = 1;" + "r2 = %[clobber_callee_saved] ll;" + "callx r2;" + "r0 += r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "exit;" + : + : __imm_addr(clobber_callee_saved) + : __clobber_all); +} + +/* r1-r5 are scratched by callx */ +SEC("socket") +__failure __msg("R1 !read_ok") +__naked void callx_scratches_args(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "r0 = r1;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* return value of the callee is tracked */ +SEC("socket") +__success __log_level(2) +__msg("to caller at 4:") +__msg("R0=6") +__naked void callx_retval_is_tracked(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long write42(void) +{ + asm volatile ( + "r2 = 42;" + "*(u64 *)(r1 + 0) = r2;" + "r0 = 0;" + "exit;" + ); +} + +/* callee writes into the stack of the caller */ +SEC("socket") +__success __retval(42) +__naked void callx_callee_writes_caller_stack(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "r2 = %[write42] ll;" + "callx r2;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm_addr(write42) + : __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type scalar, expected func") +__naked void callx_scalar(void) +{ + asm volatile ( + "r1 = 0;" + "callx r1;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R2 !read_ok") +__naked void callx_uninit_reg(void) +{ + asm volatile ( + "callx r2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R10 has type fp, expected func") +__naked void callx_fp(void) +{ + asm volatile ( + "callx r10;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type map_value, expected func") +__naked void callx_map_value(void) +{ + asm volatile ( + "r1 = %[map_array] ll;" + "r2 = r10;" + "r2 += -4;" + "r3 = 0;" + "*(u32 *)(r2 + 0) = r3;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r1 = r0;" + "callx r1;" + "1:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* the address of a subprog can't be modified before the call */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_modified_ptr(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "r2 += 8;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +SEC("socket") +__failure __msg("variable func access var_off=") +__naked void callx_variable_ptr(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r2 = %[add1] ll;" + "r2 += r0;" + "r1 = 5;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1) + : __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* only static subprogs can be called via callx */ +SEC("socket") +__failure __msg("callback function not static") +__naked void callx_global_func(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[global_add3] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(global_add3) + : __clobber_all); +} + +#define DEFINE_CALLX_RESERVED_FIELDS_PROG(NAME, SRC_REG, OFF, IMM) \ + SEC("socket") \ + __failure __msg("BPF_CALL|BPF_X uses reserved fields") \ + __naked void callx_reserved_field_ ## NAME(void) \ + { \ + asm volatile ( \ + "r1 = 5;" \ + "r2 = %[add1] ll;" \ + ".8byte %[callx_r2];" \ + "exit;" \ + : \ + : __imm_addr(add1), \ + __imm_insn(callx_r2, CALLX_INSN(BPF_REG_2, (SRC_REG), (OFF), (IMM))) \ + : __clobber_all); \ + } + +DEFINE_CALLX_RESERVED_FIELDS_PROG(src_reg, BPF_REG_1, 0, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(off, BPF_REG_0, 1, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(imm, BPF_REG_0, 0, 1) + +SEC("socket") +__failure __msg("unknown opcode 8e") +__naked void callx_jmp32(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + ".8byte %[callx32_r2];" + "exit;" + : + : __imm_addr(add1), + __imm_insn(callx32_r2, + BPF_RAW_INSN(BPF_JMP32 | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0)) + : __clobber_all); +} + +/* similar to calls of static subprogs callx is allowed under a lock */ +SEC("tc") +__success __retval(3) +__naked void callx_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r1 = 1;" + "r2 = %[add2] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(add2) + : __clobber_all); +} + +/* helpers are still not allowed under a lock in the callee */ +__naked __noinline __used +static unsigned long call_helper(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("tc") +__failure __msg("function calls are not allowed while holding a lock") +__naked void callx_helper_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r2 = %[call_helper] ll;" + "callx r2;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(call_helper) + : __clobber_all); +} + +/* self(fn) { return fn(fn); } */ +__naked __noinline __used +static unsigned long self(void) +{ + asm volatile ( + "callx r1;" + "exit;" + ); +} + +/* unbounded recursion is caught by the main verification pass */ +SEC("socket") +__failure __msg("frames is too deep") +__naked void callx_unbounded_recursion(void) +{ + asm volatile ( + "r1 = %[self] ll;" + "call self;" + "exit;" + : + : __imm_addr(self) + : __clobber_all); +} + +/* countdown(fn, n) { return n ? fn(fn, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long countdown(void) +{ + asm volatile ( + "r0 = 0;" + "if r2 == 0 goto 1f;" + "r2 += -1;" + "callx r1;" + "1:" + "exit;" + ); +} + +/* + * The depth of the recursion is known to the main verification pass, + * but recursive calls are not allowed regardless. + */ +SEC("socket") +__failure __msg("recursive call from countdown() to countdown()") +__naked void callx_bounded_recursion(void) +{ + asm volatile ( + "r1 = %[countdown] ll;" + "r2 = 2;" + "call countdown;" + "exit;" + : + : __imm_addr(countdown) + : __clobber_all); +} + +/* ping(fn1, fn2, n) { return n ? fn2(fn2, fn1, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long ping(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long pong(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +SEC("socket") +__failure __msg("recursive call from") +__naked void callx_mutual_recursion(void) +{ + asm volatile ( + "r1 = %[ping] ll;" + "r2 = %[pong] ll;" + "r3 = 3;" + "call ping;" + "exit;" + : + : __imm_addr(ping), + __imm_addr(pong) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long use_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "exit;" + ); +} + + +/* Four 480-byte frames, deeper than any budget together with their caller */ +__naked __noinline __used +static unsigned long use_stack_480_0(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_1(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_2(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_3(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_2;" + "exit;" + ); +} + +/* stack of the callee of callx is accounted */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_stack_depth(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "r2 = %[use_stack_480_3] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_480_3) + : __clobber_all); +} + +/* apply_stack_304(fn) { char buf[304]; return fn(); } */ +__naked __noinline __used +static unsigned long apply_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "callx r1;" + "exit;" + ); +} + +/* + * The address of use_stack_480_3() is taken by the main prog that doesn't + * use stack, but it is called from apply_stack_304(). + */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_stack_depth_nested(void) +{ + asm volatile ( + "r1 = %[use_stack_480_3] ll;" + "call apply_stack_304;" + "exit;" + : + : __imm_addr(use_stack_480_3) + : __clobber_all); +} + +SEC("socket") +__success __retval(0) +__naked void callx_stack_depth_ok(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 200) = r0;" + "r2 = %[use_stack_304] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long do_tail_call(void) +{ + asm volatile ( + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog) + : __clobber_all); +} + +SEC("socket") +__failure __msg("tail_calls are not allowed in functions called via callx") +__naked void callx_tail_call_in_callee(void) +{ + asm volatile ( + "r2 = %[do_tail_call] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(do_tail_call) + : __clobber_all); +} + +/* tail call in the caller of callx is fine */ +SEC("socket") +__success __retval(3) +__naked void callx_tail_call_in_caller(void) +{ + asm volatile ( + "r6 = r1;" + "r1 = 2;" + "r2 = %[add1] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog), + __imm_addr(add1) + : __clobber_all); +} + +/* read_idx(p) { return ((char *)map_value)[*p]; } */ +__naked __noinline __used +static unsigned long read_idx(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_array] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r0 += r6;" + "r0 = *(u8 *)(r0 + 0);" + "1:" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* + * Stack slots of the caller that might be read by the callee of callx + * have to be considered alive at the checkpoints before callx and + * inside of the callee. Otherwise the state with fp-8 == 1000 is pruned + * and out of bounds access in read_idx() goes unnoticed. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* in bounds access in read_idx() is fine */ +SEC("socket") +__success __retval(0) +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_ok(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* same as above, but the pointer to the stack is passed through one more frame */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_nested(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = %[read_idx] ll;" + "r2 = r10;" + "r2 += -8;" + "call apply;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* registers that are constant before callx are not constant after it */ +SEC("socket") +__success __retval(1) +__naked void callx_clobbers_const_regs(void) +{ + asm volatile ( + "r0 = 0;" + "r1 = 0;" + "r2 = %[add1] ll;" + "callx r2;" + /* dead branch pruning must not assume that r0 is still 0 */ + "if r0 == 0 goto 1f;" + "r0 = 1;" + "exit;" + "1:" + "r0 = 2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* scalar argument passed through callx is tracked precisely */ +__naked __noinline __used +static unsigned long identity(void) +{ + asm volatile ( + "r0 = r1;" + "exit;" + ); +} + +long long vals[] SEC(".data.vals") = {1, 2, 3, 4}; + +SEC("socket") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r0 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r0 stack= before 11: (bf) r0 = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 4: (8d) callx r2") +__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") +__retval(4) +__naked void callx_precision(void) +{ + asm volatile ( + "r2 = %[identity] ll;" + "r6 = 3;" + "r1 = r6;" + "callx r2;" + "r0 *= 8;" + "r1 = %[vals] ll;" + "r1 += r0;" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + : + : __imm_addr(identity), + __imm_addr(vals) + : __clobber_all); +} + +/* + * The address of a function that is never called is valid. The function is + * the last one in the program. + */ +SEC("socket") +__success __retval(6) +__naked void callx_addr_of_dead_func(void) +{ + asm volatile ( + "r6 = %[add1] ll;" + "r7 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* and when it's followed by another function */ +SEC("socket") +__success __retval(7) +__naked void callx_addr_of_dead_func_first(void) +{ + asm volatile ( + "r7 = %[add1] ll;" + "r6 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* the same without callx */ +SEC("socket") +__success __retval(6) +__naked void call_addr_of_dead_func(void) +{ + asm volatile ( + "r1 = 5;" + "call add1;" + "r7 = %[add2] ll;" + "exit;" + : + : __imm_addr(add2) + : __clobber_all); +} + +/* function pointers in C */ + +typedef int (*op_fn)(int); + +static __noinline int mul3(int x) +{ + return x * 3; +} + +static __noinline int sub7(int x) +{ + return x - 7; +} + +static __noinline int apply_op(op_fn op, int x) +{ + return op(x); +} + +SEC("socket") +__success __retval(36) +int callx_c_fn_as_arg(void *ctx) +{ + /* (5 * 3) + (28 - 7) */ + return apply_op(mul3, 5) + apply_op(sub7, 28); +} + +SEC("socket") +__success __retval(30) +int callx_c_select(void *ctx) +{ + __u32 rnd = bpf_get_prandom_u32() & 1; + op_fn op = rnd ? mul3 : sub7; + int x = rnd ? 10 : 37; + + /* 10 * 3 or 37 - 7 */ + return op(x); +} + +struct ops { + op_fn first; + op_fn second; + int bias; +}; + +static __noinline int run_ops(const struct ops *ops, int x) +{ + return ops->second(ops->first(x)) + ops->bias; +} + +SEC("socket") +__success __retval(100) +int callx_c_ops_on_stack(void *ctx) +{ + struct ops a, b; + + /* avoid an initializer with function pointers in .rodata */ + a.first = mul3; + a.second = sub7; + a.bias = 10; + b.first = sub7; + b.second = mul3; + b.bias = 1; + + /* ((7 * 3) - 7 + 10) + ((32 - 7) * 3 + 1) */ + return run_ops(&a, 7) + run_ops(&b, 32); +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c new file mode 100644 index 0000000..6cf58ca --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c
@@ -0,0 +1,903 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx through pointers to functions in read-only data */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* gcc doesn't support indirect calls */ +#if defined(__clang__) && (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + +/* + * Read-only data with pointers to functions, where the compiler puts tables + * of functions, structures of operations and vtables. libbpf resolves + * a pointer to the offset of the function in the program and the kernel + * recognizes it by that value. + */ +#define DATA(SECTION, NAME, ...) \ + ".pushsection " SECTION ",@progbits;" \ + ".balign 8;" \ + #NAME "_%=:" \ + __VA_ARGS__ \ + ".type " #NAME "_%=, @object;" \ + ".size " #NAME "_%=, .-" #NAME "_%=;" \ + ".popsection;" + +#define RODATA(NAME, ...) DATA(".rodata,\"a\"", NAME, __VA_ARGS__) + +#define FUNC_TABLE2(NAME, F0, F1) RODATA(NAME, ".quad " #F0 "; .quad " #F1 ";") + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* the second element of the table is called */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_const_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* + * The program reads the address of a function from the data, so pointers to + * functions in data are recognized only for programs that may leak pointers. + * Otherwise nothing refers to the functions. + */ +SEC("socket") +__success __retval(12) +__failure_unpriv __msg_unpriv("unreachable insn") +__caps_unpriv(CAP_BPF) +__naked void callx_rodata_needs_perfmon(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* the address of an element is used instead of the address of the table */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_elem_addr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= + 8 ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* pointers to functions are mixed with other data like in a vtable */ +SEC("socket") +__success __retval(23) +__log_level(2) +__msg("r1 = *(u64 *)(r6 +0) ; R1=7") +__msg("r2 = *(u64 *)(r6 +8) ; R2=func()") +__naked void callx_rodata_mixed_with_data(void) +{ + asm volatile ( + RODATA(vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(7) */ + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + /* add2(13) */ + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* + * Position independent code keeps constants with pointers in .data.rel.ro. + * libbpf treats it as read-only data. + */ +SEC("socket") +__success __retval(23) +__naked void callx_data_rel_ro(void) +{ + asm volatile ( + DATA(".data.rel.ro,\"aw\"", vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* an array of structures: the index selects one of the functions, not the data */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_array_of_structs(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 0x1111; .quad add2; .quad 0x2222;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r2 = arr_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* and the data of the element that the index selects can be read */ +SEC("socket") +__success __retval(16) +__naked void callx_rodata_array_of_structs_data(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r7 = arr_%= ll;" + "r7 += r3;" + "r2 = *(u64 *)(r7 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "r1 = *(u64 *)(r7 + 8);" + "r0 += r1;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* but not the data and the pointer with the same load */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_array_of_structs_any(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = arr_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* dynamic dispatch: which vtable is used is not known until run time */ +SEC("socket") +__success __retval(2) +__naked void callx_rodata_two_vtables(void) +{ + asm volatile ( + RODATA(vt_a, ".quad 1; .quad add1;") + RODATA(vt_b, ".quad 0; .quad add2;") + "call %[bpf_get_prandom_u32];" + "r6 = vt_a_%= ll;" + "r0 &= 1;" + "if r0 == 0 goto +2;" + "r6 = vt_b_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(1) or add2(0) */ + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* both functions are verified, either of them is called */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_var_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a table that has no symbol, the compiler generates such for a switch statement */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_no_symbol(void) +{ + asm volatile ( + ".pushsection .rodata,\"a\",@progbits;" + ".balign 8;" + ".Lanon_%=:" + ".quad add1;" + ".quad add2;" + ".popsection;" + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = .Lanon_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the first instruction of the function is removed by the verifier */ +__naked __noinline __used +static unsigned long nop_add1(void) +{ + asm volatile ( + "goto +0;" + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +/* the pointer follows the function when instructions are removed */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_func_starts_with_nop(void) +{ + asm volatile ( + RODATA(tbl, ".quad nop_add1; .quad 0;") + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* can't be called with a scalar in r1 */ +__naked __noinline __used +static unsigned long deref_r1(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long ret0(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +/* every function that might be called is verified */ +SEC("socket") +__failure __msg("R1 invalid mem access 'scalar'") +__naked void callx_rodata_all_callees_verified(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a function that is never called is not verified, it's dead code */ +SEC("socket") +__success __retval(0) +__naked void callx_rodata_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* a pointer that is read and never called doesn't keep the function, it's NULL */ +SEC("socket") +__success __retval(1) +__naked void callx_rodata_read_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 0;" + "callx r2;" + "r2 = *(u64 *)(r6 + 8);" + "if r2 != 0 goto +1;" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* the index may select what is not a pointer to a function */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_index_beyond_table(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0x1234; .quad 0x5678;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the address of a function is not known until the program is jitted */ +SEC("socket") +__failure __msg("read of 4 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_partial_read(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 10;" + "callx r2;" + "r0 = *(u32 *)(r6 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("read of 8 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_misaligned_read(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 4);" + "exit;" + ::: __clobber_all); +} + +/* + * The kernel doesn't look for pointers to functions in the data of a program + * that doesn't have callx. It's a number there. + */ +SEC("socket") +__success +__naked void callx_rodata_no_callx(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r0 = *(u32 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* the data next to a pointer is still known to the verifier */ +SEC("socket") +__success __retval(0x1234) +__log_level(2) +__msg("R0=4660") +__naked void callx_rodata_data_is_const(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad 0x1234;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 8);" + "exit;" + ::: __clobber_all); +} + +/* + * The map has the offset of the function until the program is jitted. + * bpf_compute_const_regs() must not take it for a constant, otherwise + * bpf_prune_dead_branches() turns the branch below into the wrong one. + */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_ptr_is_not_const(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r0 = 0;" + /* the offset is less, the address is greater */ + "if r2 < 0x100000 goto 1f;" + "r1 = 10;" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +/* the pointer read from the data can't be modified */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_rodata_modified_ptr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r2 += 8;" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* + * A pointer to a global function is not recognized, it's a number. The function + * is called directly as well, so that libbpf adds it to the program and stores + * its offset into the table. + */ +SEC("socket") +__failure __msg("R2 has type scalar, expected func") +__naked void callx_rodata_global_func(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, global_add3) + "r1 = 1;" + "call global_add3;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* selfcall(n) { return n ? tbl[0](n - 1) : 0; }, where tbl[0] == selfcall */ +__naked __noinline __used +static unsigned long selfcall(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, selfcall, ret0) + "r0 = 0;" + "if r1 == 0 goto 1f;" + "r1 += -1;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("recursive call from selfcall() to selfcall()") +__naked void callx_rodata_recursion(void) +{ + asm volatile ( + "r1 = 2;" + "call selfcall;" + "exit;" + ::: __clobber_all); +} + + +/* Four 480-byte frames, deeper than any budget together with their caller */ +__naked __noinline __used +static unsigned long use_stack_480_0(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_1(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_2(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_3(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_2;" + "exit;" + ); +} + +/* stack of all possible callees is accounted */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_rodata_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, use_stack_480_3) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* stack of a callback that is read from the data is accounted too */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_rodata_callback_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, use_stack_480_3, ret0) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "r6 = tbl_%= ll;" + "r1 = 1;" + "r2 = *(u64 *)(r6 + 0);" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_loop) + : __clobber_all); +} + +static __noinline __used int pull_data(struct __sk_buff *skb) +{ + return bpf_skb_pull_data(skb, 0); +} + +static __noinline __used int calls_pull_data(struct __sk_buff *skb) +{ + return pull_data(skb) + 1; +} + +__noinline __used int global_calls_pull_data(struct __sk_buff *skb) +{ + return calls_pull_data(skb) + 2; +} + +/* + * check_cfg() explores the functions that the data points to when it sees + * the address of the data. It must not lose the effects of the functions + * that call each other: global_calls_pull_data() changes packet data. + */ +SEC("tc") +__failure __msg("R7 invalid mem access 'scalar'") +__naked void callx_rodata_callee_effects(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, pull_data, calls_pull_data) + "r6 = r1;" + "r7 = *(u32 *)(r6 + %[data]);" + "r8 = *(u32 *)(r6 + %[data_end]);" + "r2 = r7;" + "r2 += 8;" + "if r2 > r8 goto 1f;" + "r1 = r6;" + "call global_calls_pull_data;" + "r0 = *(u8 *)(r7 + 0);" + "1:" + "r9 = tbl_%= ll;" + "r2 = *(u64 *)(r9 + 8);" + "r1 = r6;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* pointers to functions in C */ + +typedef int (*op_fn)(int); + +#define DEFINE_OP(N) static __noinline int op##N(int x) { return x * (N + 2) + N; } + +DEFINE_OP(0) DEFINE_OP(1) DEFINE_OP(2) DEFINE_OP(3) +DEFINE_OP(4) DEFINE_OP(5) DEFINE_OP(6) DEFINE_OP(7) +DEFINE_OP(8) DEFINE_OP(9) DEFINE_OP(10) DEFINE_OP(11) +DEFINE_OP(12) DEFINE_OP(13) DEFINE_OP(14) DEFINE_OP(15) + +static op_fn const ops[] = { + op0, op1, op2, op3, op4, op5, op6, op7, + op8, op9, op10, op11, op12, op13, op14, op15, +}; + +int op_idx = 11; + +SEC("socket") +__success __retval(50) +int callx_c_table(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* op11(3) = 3 * 13 + 11 */ + return ops[i](3); +} + +SEC("socket") +__success __retval(50) +int callx_c_table_null_check(void *ctx) +{ + unsigned int i = op_idx; + op_fn op; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* the address of a function is not known until the program is jitted */ + op = ops[i]; + if (!op) + return -2; + return op(3); +} + +/* a structure of operations, where pointers to functions are mixed with data */ +struct shape_ops { + int id; + op_fn area; + long scale; + op_fn perimeter; +}; + +static const struct shape_ops square_ops = { 1, op1, 10, op2 }; +static const struct shape_ops circle_ops = { 2, op3, 20, op1 }; + +static __noinline int use_shape(const struct shape_ops *ops, int x) +{ + return ops->area(x) * ops->scale + ops->perimeter(ops->id); +} + +SEC("socket") +__success __retval(433) +int callx_c_ops_mixed_with_data(void *ctx) +{ + /* + * square: op1(2) * 10 + op2(1) = 7 * 10 + 6 = 76 + * circle: op3(3) * 20 + op1(2) = 18 * 20 + 7 = 367 + * minus 10, since op_idx is 11 + */ + return use_shape(&square_ops, 2) + use_shape(&circle_ops, 3) - (op_idx == 11 ? 10 : 0); +} + +/* the ops are selected at run time */ +SEC("socket") +__success __retval(367) +int callx_c_ops_selected(void *ctx) +{ + const struct shape_ops *ops = op_idx == 11 ? &circle_ops : &square_ops; + + return use_shape(ops, 3); +} + +/* + * clang-22 with -mcpu=v4 drops the lookup table it builds for this switch + * ("Undefined temporary symbol .Lswitch.table.call_by_switch"). + * Fixed in clang-23 by llvm commit 9fc458a1d95b. + * + * https://github.com/llvm/llvm-project/commit/9fc458a1d95b + */ +#if __clang_major__ != 22 || __BPF_CPU_VERSION__ < 4 +/* the compiler might turn the switch into a table that has no symbol */ +static __noinline int call_by_switch(unsigned int idx, int x) +{ + op_fn op; + + switch (idx) { + case 0: + op = op8; + break; + case 1: + op = op9; + break; + case 2: + op = op10; + break; + case 3: + op = op11; + break; + case 4: + op = op12; + break; + case 5: + op = op13; + break; + case 6: + op = op14; + break; + case 7: + op = op15; + break; + default: + return -1; + } + return op(x); +} + +SEC("socket") +__success __retval(50) +int callx_c_switch_table(void *ctx) +{ + /* op11(3) = 3 * 13 + 11 */ + return call_by_switch(op_idx - 8, 3); +} + +#else + +SEC("socket") +__skip("clang-22 bug with -mcpu=v4, switch table test cannot be built") +__success +int callx_c_switch_table(void *ctx) +{ + return 0; +} + +#endif + +/* + * Misaligned pointers to functions are ignored, the rest of the data is + * accessible as before. The program doesn't have callx, so libbpf doesn't + * append the functions that .rodata points to and doesn't make a copy of + * .rodata for it. + */ +static const struct { + char tag; + op_fn op; +} __attribute__((packed)) packed_ops[] = { + { 5, op0 }, + { 7, op1 }, +}; + +SEC("socket") +__success __retval(7) +int callx_c_packed_struct(void *ctx) +{ + return packed_ops[op_idx & 1].tag; +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index 3c3bb03..b429fa7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c
@@ -56,6 +56,46 @@ __naked void out_of_range_jump2(void) " ::: __clobber_all); } +static __naked __noinline __used int cross_subprog_target(void) +{ + asm volatile (" \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("jump across subprogram boundary") +__failure __msg("jump out of range from insn 1") +__msg("jump_across_subprog_boundary @ verifier_cfg.c") +__naked void jump_across_subprog_boundary(void) +{ + asm volatile (" \ + call cross_subprog_target; \ + goto +1; \ + exit; \ +" ::: __clobber_all); +} + +static __naked __noinline __used int fallthrough_subprog(void) +{ + asm volatile (" \ + r0 = 0; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("subprogram fallthrough") +__failure __msg("last insn is not an exit or jmp") +__msg("fallthrough_subprog @ verifier_cfg.c") +__naked void subprog_fallthrough(void) +{ + asm volatile (" \ + call fallthrough_subprog; \ + exit; \ +" ::: __clobber_all); +} + SEC("socket") __description("invalid DW LDSX instruction in diagnostics") __failure __msg("BUG_ldx_99") @@ -173,4 +213,43 @@ l1_%=: exit; \ " ::: __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} map_array SEC(".maps"); + +extern void bpf_throw(__u64 cookie) __ksym; + +/* + * The subprogram leaves through the tail call or the exception and never + * reaches an exit insn, so the hidden edge of the tail call has no exit to + * point at. The stack access makes the liveness pass walk the subprogram, + * and the loop compiles to a single jump back onto itself. + */ +__noinline __used +static int tail_call_or_throw_subprog(void *ctx) +{ + volatile __u32 idx = 0; + + bpf_tail_call(ctx, &map_array, idx); + bpf_throw(0); + for (;;) {} + return 0; +} + +SEC("tc") +__description("tail call in a subprogram without an exit") +__arch_x86_64 +__arch_arm64 +__success +__naked void tail_call_in_subprog_without_exit(void) +{ + asm volatile (" \ + call tail_call_or_throw_subprog; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_ctx.c b/tools/testing/selftests/bpf/progs/verifier_ctx.c index 7856dad..9d42ba8 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ctx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ctx.c
@@ -208,7 +208,7 @@ __naked void null_check_7_ctx_bind(void) SEC("cgroup/post_bind4") __description("pass ctx or null check, 8: null (bind)") -__failure __msg("R1 type=scalar expected=ctx") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void null_check_8_null_bind(void) { asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index dcc2dd4..4c55b70 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c
@@ -87,7 +87,7 @@ __weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted) SEC("?kprobe") __failure __log_level(2) -__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") +__msg("Possibly NULL pointer passed to trusted R1") __msg("Caller passes invalid args into func#1 ('subprog_trusted_task_nonnull')") int trusted_task_arg_nonnull_fail1(void *ctx) { @@ -96,7 +96,7 @@ int trusted_task_arg_nonnull_fail1(void *ctx) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("R1 type=trusted_ptr_or_null_ expected=ptr_, trusted_ptr_, rcu_ptr_") +__msg("Possibly NULL pointer passed to trusted R1") __msg("Caller passes invalid args into func#1 ('subprog_trusted_task_nonnull')") int trusted_task_arg_nonnull_fail2(void *ctx) { @@ -177,7 +177,7 @@ __weak int subprog_trusted_destroy(struct task_struct *task __arg_trusted) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(trusted_destroy_fail, struct task_struct *task, u64 clone_flags) { return subprog_trusted_destroy(task); @@ -289,6 +289,41 @@ __weak int subprog_void_untrusted(void *p __arg_untrusted) return *(int *)p; } +__weak int subprog_trusted_bare(struct task_struct *task __arg_trusted) +{ + return task->pid; +} + +SEC("tp_btf/task_newtask") +__failure +__msg("R1 must be referenced or trusted") +__msg("Caller passes invalid args into func#{{.*}} ('subprog_trusted_bare')") +int bare_to_trusted(void *ctx) +{ + struct task_struct *cur = bpf_get_current_task_btf(); + struct task_struct *wakee; + + if (!cur) + return 0; + wakee = cur->last_wakee; + if (!wakee) + return 0; + return subprog_trusted_bare(wakee); +} + +SEC("tp_btf/task_newtask") +__failure +__msg("R1 must be referenced or trusted") +__msg("Caller passes invalid args into func#{{.*}}") +int memrcu_to_trusted(void *ctx) +{ + struct task_struct *cur = bpf_get_current_task_btf(); + + if (!cur) + return 0; + return subprog_trusted_task_nullable(cur->real_parent); +} + __weak int subprog_char_untrusted(char *p __arg_untrusted) { return *(int *)p; @@ -389,7 +424,8 @@ __weak int subprog_pkt_ptr_changes_data(struct __sk_buff *skb __arg_ctx, SEC("?tc") __failure __log_level(2) -__msg("R2 is a packet pointer, but func#{{[0-9]+}} may change packet data") +__msg("cannot pass packet pointer R2") +__msg("function may change packet data") __msg("Caller passes invalid args into func#{{[0-9]+}} ('subprog_pkt_ptr_changes_data')") int pkt_ptr_to_global_mem_arg_changes_data(struct __sk_buff *skb) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 966f493..574b26b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c
@@ -195,7 +195,7 @@ int arg_tag_nonnull_ptr_good(void *ctx) SEC("?raw_tp") __failure __log_level(2) -__msg("R1 is expected to be non-NULL") +__msg("Possibly NULL pointer passed to trusted R1") int arg_tag_nonnull_ptr_null_bad(void *ctx) { int y = 74; @@ -536,4 +536,35 @@ int return_from_void_global(struct __sk_buff *skb) return 0; } +int global_calls_loop(int x); + +static __noinline int static_calls_global(int x) +{ + return global_calls_loop(x); +} + +static __noinline int loop_cb_calls_static(u32 i, void *ctx) +{ + return static_calls_global(i); +} + +__noinline int global_calls_loop(int x) +{ + bpf_loop(1, loop_cb_calls_static, NULL, 0); + return 0; +} + +/* + * loop_cb_calls_static() -> static_calls_global() -> global_calls_loop() -> + * bpf_loop() -> loop_cb_calls_static() is an unbounded recursion that the + * main verification pass can't see, because it doesn't follow calls of global + * functions. None of the functions use stack. + */ +SEC("?raw_tp") +__failure __msg("recursive call from global_calls_loop() to loop_cb_calls_static()") +int recursion_via_global_func_and_callback(const void *ctx) +{ + return loop_cb_calls_static(0, NULL); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_gotol.c b/tools/testing/selftests/bpf/progs/verifier_gotol.c index d5d8f24..c0fd30a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotol.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotol.c
@@ -52,6 +52,7 @@ __naked void gotol_large_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 0e27c2c..3d1ff70 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c
@@ -6,7 +6,9 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_powerpc) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) #define DEFINE_SIMPLE_JUMP_TABLE_PROG(NAME, SRC_REG, OFF, IMM, OUTCOME) \ \ @@ -101,9 +103,11 @@ __naked void jump_table_terminal_gotox_subprog(void) */ SEC("socket") __failure __msg("no jump tables found for subprog starting at 0") +__msg(">>> 1 | (0d) gotox r0") __naked void jump_table_no_jump_table(void) { asm volatile (" \ + r0 = 0; \ .8byte %[gotox_r0]; \ r0 = 1; \ exit; \ @@ -366,7 +370,7 @@ __used static int test_subprog(void) } SEC("socket") -__failure __msg("jump table for insn 4 points outside of the subprog [0,10]") +__failure __msg("jump table of subprog starting at 0 spans multiple subprogs") __naked void jump_table_outside_subprog(void) { asm volatile (" \ @@ -457,6 +461,133 @@ __naked void spill_fill_ptr_to_insn(void) : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ +/* + * Emit each table entry next to the instruction it targets. Numeric labels + * can be reused by .rept, while %= keeps table symbols unique per asm block. + * BASE is the code section: libbpf relocates section-relative byte offsets + * to instruction offsets in the loaded program. + */ +#define GOTOX_TABLE_BEGIN(COUNT) \ + ".pushsection .jumptables,\"\",@progbits;" \ + "jt_%=:;" \ + ".global jt_%=;" \ + ".size jt_%=, 8 * (" #COUNT ");" \ + ".popsection;" \ + "r1 = jt_%= ll;" + +#define GOTOX_TABLE_ENTRY(LABEL, BASE) \ + ".pushsection .jumptables,\"\",@progbits;" \ + ".quad " LABEL " - " BASE ";" \ + ".popsection;" + +/* N gotox instructions, each targeting all N gotox plus the final block. */ +#define GOTOX_SELF_TARGETS(N, BASE) \ + GOTOX_TABLE_BEGIN(N + 1) \ + "r0 = 0;" \ + ".rept " #N ";" \ + GOTOX_TABLE_ENTRY("1f", BASE) \ + "1:; .8byte %[gotox];" \ + ".endr;" \ + GOTOX_TABLE_ENTRY("2f", BASE) \ + "2:;" + +#define GOTOX_OPERAND \ + __imm_insn(gotox, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + +/* + * Count edges even when their targets are already discovered. These + * programs deliberately use a scalar: the CFG edge limit must reject them + * before symbolic verification reaches the invalid gotox operand. + */ +SEC("socket") +__description("too-many-gotox-edges") +__failure __msg("number of indirect jump edges in the program exceeds") +__naked void too_many_gotox_edges(void) +{ + asm volatile ( + /* 1000 * 1001 = 1,001,000 edges. */ + GOTOX_SELF_TARGETS(1000, "socket") + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +SEC("socket") +__description("gotox-edges-at-limit") +__failure __msg("Too many nested loops") +__naked void gotox_edges_at_limit(void) +{ + asm volatile ( + /* + * 1000 gotox * 1000 targets = 1,000,000 CFG edges. At run + * time, each block loads the next table entry and jumps to + * the following block, so the program terminates. + * + * Every block can jump to every later block, so the CFG is + * a nest of ~1000 irreducible loops, deeper than the + * verifier's loop stack; the program is rejected. + */ + GOTOX_TABLE_BEGIN(1000) + ".rept 1000;" + GOTOX_TABLE_ENTRY("1f", "socket") + "r0 = *(u64 *)(r1 + 0);" + "r1 += 8;" + ".8byte %[gotox];" + "1:;" + ".endr;" + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +static __noinline __used __naked void gotox_edges_subprog(void) +{ + asm volatile ( + GOTOX_SELF_TARGETS(750, ".text") + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +SEC("socket") +__description("gotox-edges-across-subprogs") +__failure __msg("number of indirect jump edges in the program exceeds") +__naked void gotox_edges_across_subprogs(void) +{ + asm volatile ( + /* Each subprog has 563,250 edges; together, 1,126,500. */ + GOTOX_SELF_TARGETS(750, "socket") + "call gotox_edges_subprog;" + "exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +/* + * The gotox target is a ja +0, which the nop removal pass must keep as the + * jump table still names it. + */ +SEC("socket") +__success __retval(0) +__xlated("goto pc+0") +__naked void jump_table_target_nop(void) +{ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad nop_%= - socket; \ + .size jt0_%=, 8; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 = *(u64 *)(r0 + 0); \ + .8byte %[gotox_r0]; \ +nop_%=: \ + goto +0; \ + r0 = 0; \ + exit; \ +" : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} + +#endif /* gotox: x86, arm64, powerpc, riscv64 */ char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index 343fc08..2782faf 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c
@@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ /* Converted from tools/testing/selftests/bpf/verifier/helper_access_var_len.c */ #include <linux/bpf.h> @@ -621,7 +622,7 @@ l0_%=: exit; \ SEC("tracepoint") __description("helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)") -__failure __msg("R1 type=scalar expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void ptr_to_mem_or_null_8(void) { asm volatile (" \ @@ -637,7 +638,7 @@ __naked void ptr_to_mem_or_null_8(void) SEC("tracepoint") __description("helper access to variable memory: size > 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)") -__failure __msg("R1 type=scalar expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void ptr_to_mem_or_null_9(void) { asm volatile (" \ @@ -822,4 +823,335 @@ __naked void bytes_no_leak_init_memory(void) : __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_WRONLY_PROG); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_wo SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_rw SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_RDONLY_PROG); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_ro SEC(".maps"); + +SEC("tc") +__failure __msg("read from map forbidden") +int writeonly_sized_input(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_wo, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("tc") +__success +int readwrite_sized_input(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_rw, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("tc") +__failure __msg("write into map forbidden") +int readonly_sized_output(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_ro, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("xdp") +__failure __msg("read from map forbidden") +int xdp_writeonly_sized_input(struct xdp_md *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_wo, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return XDP_PASS; +} + +SEC("sockops") +__failure __msg("read from map forbidden") +int writeonly_header_option(struct bpf_sock_ops *ctx) +{ + struct bpf_fib_lookup *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_load_hdr_opt(ctx, buf, sizeof(*buf), 0); + return 0; +} + +SEC("sockops") +__success +int readwrite_header_option(struct bpf_sock_ops *ctx) +{ + struct bpf_fib_lookup *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_rw, &key); + if (buf) + bpf_load_hdr_opt(ctx, buf, sizeof(*buf), 0); + return 0; +} + +SEC("tc") +__success +int snprintf_writeonly_output(struct __sk_buff *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_snprintf(buf, 16, "ok", NULL, 0); + return 0; +} + +SEC("tc") +__failure __msg("write into map forbidden") +int snprintf_readonly_output(struct __sk_buff *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_ro, &key); + if (buf) + bpf_snprintf(buf, 16, "ok", NULL, 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +int sysctl_writeonly_output(struct bpf_sysctl *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_sysctl_get_name(ctx, buf, 16, 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_initialized_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_uninitialized_stack(void) +{ + asm volatile ( + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = 0;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_partial_initialized_bytes(void) +{ + asm volatile ( + "*(u32 *)(r10 - 16) = 0;" + "*(u8 *)(r10 - 1) = 1;" + "goto +0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -1+0 size 1") +__naked void sysctl_partial_invalid_bytes(void) +{ + asm volatile ( + "*(u32 *)(r10 - 16) = 0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_partial_variable_size(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 15;" + "r3 += 1;" + "*(u8 *)(r10 - 1) = 1;" + "goto +0;" + "r2 = r10;" + "r2 += -16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -1+0 size 1") +__naked void sysctl_partial_variable_invalid(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 15;" + "r3 += 1;" + "r2 = r10;" + "r2 += -16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack R2") +__naked void sysctl_partial_spilled_pointer(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "goto +0;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = 0;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack R2") +int sysctl_partial_dynptr(struct bpf_sysctl *ctx) +{ + struct bpf_dynptr ptr; + long long key = 0, *data; + + data = bpf_map_lookup_elem(&map_hash_8b, &key); + if (!data) + return 0; + bpf_dynptr_from_mem(data, sizeof(*data), 0, &ptr); + bpf_sysctl_get_name(ctx, (char *)&ptr, sizeof(ptr), 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__naked void sysctl_partial_variable_offset(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 8;" + "r2 += r10;" + "r2 += -24;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 16);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +int snprintf_partial_output_runtime(struct __sk_buff *ctx) +{ + char buf[16]; + + buf[15] = 42; + bpf_snprintf(buf, sizeof(buf), "ok", NULL, 0); + return buf[15]; +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c index 71cee3f..12786b7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c
@@ -258,7 +258,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test11, cls unsuitable helper 1") -__failure __msg("helper access to the packet") +__failure __msg("function access to the packet") __naked void test11_cls_unsuitable_helper_1(void) { asm volatile (" \ @@ -283,7 +283,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test12, cls unsuitable helper 2") -__failure __msg("helper access to the packet") +__failure __msg("function access to the packet") __naked void test12_cls_unsuitable_helper_2(void) { asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_idoms.c b/tools/testing/selftests/bpf/progs/verifier_idoms.c new file mode 100644 index 0000000..fdee6bc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_idoms.c
@@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* + * The immediate dominator is printed by log_program() as the numeric column + * immediately before the "<insn#>:" field of every "Program dump" line: + * + * Program dump (scc? loop_header? idom insn#: live_regs_before): + * -1 0: ....... (b7) r0 = 0 <- idom(0) = -1 (subprog entry) + * 0 1: ....... (07) r0 += 1 <- idom(1) = 0 + * ^^^^^^ + * idom insn# + */ + +/* Straight-line code, with an ldimm64 in the middle. */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (18) r1 = 0x1122334455667788") +__msg("{{.*}} 1 3: {{.*}} (07) r0 += 1") +__msg("{{.*}} 3 4: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 4 5: {{.*}} (95) exit") +__naked void straight_line_ldimm64(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = 0x1122334455667788 ll; \ + r0 += 1; \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Asymmetric if-then-else diamond: the "then" arm is 5 instructions long, the + * "else" arm is a single instruction. The merge point (insn 8) is dominated by + * the branch (insn 1), not by either arm. This forces idoms_intersect() to walk + * the two predecessors up unequal postorder depths before they meet. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (25) if r0 > 0x0 goto pc+5") +__msg("{{.*}} 1 2: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 2 3: {{.*}} (b7) r1 = 2") +__msg("{{.*}} 3 4: {{.*}} (b7) r1 = 3") +__msg("{{.*}} 4 5: {{.*}} (b7) r1 = 4") +__msg("{{.*}} 5 6: {{.*}} (05) goto pc+1") +__msg("{{.*}} 1 7: {{.*}} (b7) r1 = 9") +__msg("{{.*}} 1 8: {{.*}} (95) exit") +__naked void asymmetric_diamond(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r1 = 1; \ + r1 = 2; \ + r1 = 3; \ + r1 = 4; \ + goto 2f; \ +1: r1 = 9; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Simple loop: the header (insn 1) is dominated by the pre-header (insn 0). The + * back edge (insn 2 -> insn 1) must not change idom(1); the back-edge + * predecessor folds up via idoms_intersect(). + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg("{{.*}} 2 3: {{.*}} (95) exit") +__naked void simple_loop(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Loop with an if-else in its body. The in-loop merge point (insn 5) is + * dominated by the in-loop branch (insn 3), combining a back-edge intersect + * with a forward-diamond intersect. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (bf) r1 = r0") +__msg("{{.*}} 2 3: {{.*}} (25) if r1 > 0x5 goto pc+1") +__msg("{{.*}} 3 4: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 3 5: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 5 6: {{.*}} (a5) if r0 < 0x64 goto pc-6") +__msg("{{.*}} 6 7: {{.*}} (95) exit") +__naked void loop_with_if_else_body(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = r0; \ + if r1 > 5 goto 2f; \ + r1 = 1; \ +2: r0 += r1; \ + if r0 < 100 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Nested loops: the idom chain runs inner-header -> outer-body -> outer-header + * -> pre-header. Exercises intersect across back edges at two nesting depths. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 2 3: {{.*}} (07) r1 += 1") +__msg("{{.*}} 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") +__msg("{{.*}} 4 5: {{.*}} (a5) if r0 < 0xa goto pc-5") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__naked void nested_loops(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Loop header (insn 1) with two back edges (from insn 4 and insn 5), i.e. three + * predecessors. Both back edges route through the incrementing header, so the + * loop is bounded and verifies. Repeated idoms_intersect() at the header must + * stay stable and keep idom(1) at the pre-header (insn 0). + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r6 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r6 += 1") +__msg("{{.*}} 1 2: {{.*}} (25) if r6 > 0xa goto pc+3") +__msg("{{.*}} 2 3: {{.*}} (bf) r7 = r6") +__msg("{{.*}} 3 4: {{.*}} (a5) if r7 < 0x5 goto pc-4") +__msg("{{.*}} 4 5: {{.*}} (05) goto pc-5") +__msg("{{.*}} 2 6: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 6 7: {{.*}} (95) exit") +__naked void multi_backedge_header(void) +{ + asm volatile (" \ + r6 = 0; \ +1: r6 += 1; \ + if r6 > 10 goto 2f; \ + r7 = r6; \ + if r7 < 5 goto 1b; \ + goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Irreducible CFG (loop with two entries, insn 3 and insn 5, reached from the + * insn 2 branch). Dominators remain well-defined; this is a convergence test + * for the fixpoint under irreducibility. Note insn 6 is dominated by the branch + * (insn 2), not by either arm. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 1 2: {{.*}} (25) if r0 > 0x5 goto pc+2") +__msg("{{.*}} 2 3: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 3 4: {{.*}} (05) goto pc+1") +__msg("{{.*}} 2 5: {{.*}} (b7) r1 = 2") +__msg("{{.*}} 2 6: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 6 7: {{.*}} (a5) if r0 < 0x10 goto pc-5") +__msg("{{.*}} 7 8: {{.*}} (95) exit") +__naked void irreducible(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ + if r0 > 5 goto 2f; \ +1: r1 = 1; \ + goto 3f; \ +2: r1 = 2; \ +3: r0 += r1; \ + if r0 < 16 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Idoms are computed per subprog. Layout (libbpf preorder) is: + * main: 0..1, sub: 2..7 + * The callee entry (insn 2) must have idom -1 (subprog reset), and idoms inside + * the callee must reference only the callee's instructions, never main's. The + * in-callee merge (insn 7) is dominated by the in-callee branch (insn 3). The + * branch is on a prandom value so it is not constant-folded away. + */ +static __naked __noinline __used +unsigned long idoms_diamond_sub(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r0 = 1; \ + goto 2f; \ +1: r0 = 2; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call pc+1") +__msg("{{.*}} 0 1: {{.*}} (95) exit") +__msg("{{.*}} -1 2: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 2 3: {{.*}} (25) if r0 > 0x0 goto pc+2") +__msg("{{.*}} 3 4: {{.*}} (b7) r0 = 1") +__msg("{{.*}} 4 5: {{.*}} (05) goto pc+1") +__msg("{{.*}} 3 6: {{.*}} (b7) r0 = 2") +__msg("{{.*}} 3 7: {{.*}} (95) exit") +__naked void multi_subprog(void) +{ + asm volatile (" \ + call idoms_diamond_sub; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The first instruction of a subprog (insn 3) is itself a loop header, i.e. it + * has an incoming back edge. Its idom must still be -1 (the subprog entry has no + * dominator). Exercises the idoms[start]=start ... idoms[start]=-1 handling in + * compute_subprog_idoms(). + */ +static __naked __noinline __used +unsigned long idoms_entry_header_sub(void) +{ + asm volatile (" \ +1: r1 += 1; \ + if r1 < 10 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 0 1: {{.*}} (85) call pc+1") +__msg("{{.*}} 1 2: {{.*}} (95) exit") +__msg("{{.*}} -1 3: {{.*}} (07) r1 += 1") +__msg("{{.*}} 3 4: {{.*}} (a5) if r1 < 0xa goto pc-2") +__msg("{{.*}} 4 5: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__naked void entry_is_loop_header(void) +{ + asm volatile (" \ + r1 = 0; \ + call idoms_entry_header_sub; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Self-loop: insn 1 is its own predecessor. idom(1) must be the pre-header + * (insn 0); the self-edge is ignored (idoms[pred] == -1 on the first pass, then + * idoms_intersect(a == b) short-circuits). Program is rejected later for an + * infinite loop, but the dump (and idoms) are printed before that. + */ +SEC("socket") +__failure +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (a5) if r0 < 0xa goto pc-1") +__msg("{{.*}} 1 2: {{.*}} (95) exit") +__msg("infinite loop detected") +__naked void self_loop(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ +1: if r0 < 10 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) +/* + * Indirect jump (gotox) with a two-entry jump table. The gotox (insn 4) has two + * successors (insn 5 and insn 7), so both targets have the gotox as their only + * predecessor and idom. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (18) r0 = {{0x[0-9a-f]+}}") +__msg("{{.*}} 0 2: {{.*}} (07) r0 += 8") +__msg("{{.*}} 2 3: {{.*}} (79) r0 = *(u64 *)(r0 +0)") +__msg("{{.*}} 3 4: {{.*}} (0d) gotox r0") +__msg("{{.*}} 4 5: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__msg("{{.*}} 4 7: {{.*}} (b7) r0 = 1") +__msg("{{.*}} 7 8: {{.*}} (95) exit") +__naked void gotox_jump_table(void) +{ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad ret0_%= - socket; \ + .quad ret1_%= - socket; \ + .size jt0_%=, 16; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 += 8; \ + r0 = *(u64 *)(r0 + 0); \ + .8byte %[gotox_r0]; \ +ret0_%=: \ + r0 = 0; \ + exit; \ +ret1_%=: \ + r0 = 1; \ + exit; \ +" : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c b/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c index 9f3f2b7..4fd7173 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c
@@ -68,6 +68,13 @@ __naked void btf_jit_convergence_je_jmp(void) "r3 &= 0x1;" "r2 *= 0xa8;" "if r3 == 0x0 goto l2_%=;" + /* + * The x86-64 code from the je to l0 up to l0 has to be 125 bytes with + * the last goto l9 at two bytes and 128 with it at five, so that the + * je fits into an imm8 only while the jmp does not and the two keep + * flipping each other's size. The mov pads the region to that. + */ + "r3 = 0x1234;" "r1 += r2;" "r1 -= r7;" "r1 += 0x8;"
diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c index 02e562f..5b92da2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c
@@ -11,7 +11,7 @@ __jited(" addq %gs:{{.*}}, %rax") __arch_arm64 __jited(" mrs x8, SP_EL0") __arch_riscv64 -__jited(" mv a5, tp") +__jited(" mv t6, tp") __arch_loongarch __jited(" move $a5, $tp") int inline_bpf_get_current_task(void)
diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c new file mode 100644 index 0000000..8800956 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c
@@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, + struct bpf_sock_tuple *bpf_tuple, + u32 tuple__sz, struct bpf_ct_opts *opts, + u32 opts__sz) __ksym; +void bpf_ct_release(struct nf_conn *nfct) __ksym; + +char _license[] SEC("license") = "GPL"; + +SEC("tc") +__description("kfunc packet write requests writable skb") +__success +/* bpf_unclone_prologue() */ +__xlated("r6 = *(u8 *)(r1 +{{[0-9]+}})") +__xlated("...") +__xlated("w6 &= {{(1|128)}}") +__xlated("...") +__xlated("if r6 == 0x0 goto") +__xlated("r6 = r1") +__xlated("r2 ^= r2") +__xlated("call") +__xlated("if r0 == 0x0 goto") +__xlated("w0 = 2") +__xlated("...") +__xlated("exit") +__xlated("r1 = r6") +int kfunc_packet_write(struct __sk_buff *skb) +{ + void *data_end = (void *)(long)skb->data_end; + void *data = (void *)(long)skb->data; + struct bpf_sock_tuple tuple = {}; + struct nf_conn *nfct; + + if (data + sizeof(struct bpf_ct_opts) > data_end) + return 0; + + /* An invalid tuple size makes bpf_skb_ct_lookup() write opts->error. */ + nfct = bpf_skb_ct_lookup(skb, &tuple, 1, data, sizeof(struct bpf_ct_opts)); + if (nfct) + bpf_ct_release(nfct); + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c new file mode 100644 index 0000000..20ed71a5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c
@@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* Keep the kfunc BTF records used by the inline assembly. */ +void __kfunc_btf_root(void) +{ + asm volatile ("" : + : "r"(&bpf_kfunc_test_uninit_struct), + "r"(&bpf_kfunc_test_uninit_mem), + "r"(&bpf_kfunc_test_uninit_partial), + "r"(&bpf_kfunc_test_uninit_alias)); +} + +SEC("tc") +__success __retval(10) +__log_level(2) +__msg("call bpf_kfunc_test_uninit_struct{{.*}}; def: fp0-8 fp0-16") +__msg_unpriv("call bpf_kfunc_test_uninit_struct{{.*}}; use: fp0-8 fp0-16") +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void struct_poisoned_at_checkpoint(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u32 *)(r10 - 12);" + "r0 += r1;" + "r1 = *(u32 *)(r10 - 8);" + "r0 += r1;" + "r1 = *(u32 *)(r10 - 4);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(0x2a2a2a2a) +__log_level(2) +__msg("call bpf_kfunc_test_uninit_mem{{.*}}; def: fp0-8") +__msg_unpriv("call bpf_kfunc_test_uninit_mem{{.*}}; use: fp0-8") +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void sized_buffer_poisoned_at_checkpoint(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u32 *)(r10 - 8);" + "r1 = *(u32 *)(r10 - 4);" + "if r0 == r1 goto +1;" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(7) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void initialized_input_alias(void) +{ + asm volatile ( + "*(u32 *)(r10 - 8) = 7;" + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call %[bpf_kfunc_test_uninit_alias];" + "exit;" + : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); +} + +SEC("tc") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack") +__naked void uninitialized_input_alias(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call %[bpf_kfunc_test_uninit_alias];" + "exit;" + : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); +} + +SEC("tc") +__success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void uninitialized_struct_output_ignored(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void variable_size_uninitialized_output_ignored(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(49) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void partial_output_preserves_initialized_bytes(void) +{ + asm volatile ( + "*(u8 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 1) = 7;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_partial];" + "r0 = *(u8 *)(r10 - 8);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_partial) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 1") +__naked void uninitialized_sized_output_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u8 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 1") +__naked void variable_size_uninitialized_output_read(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u8 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(1) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -16+0 size 4") +__naked void fixed_struct_uninitialized_output_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = *(u32 *)(r10 - 16);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(49) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void partial_output_variable_size(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "*(u8 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 1) = 7;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_partial];" + "r0 = *(u8 *)(r10 - 8);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_partial) : __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c new file mode 100644 index 0000000..ab3bb81 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c
@@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* Keep the kfunc BTF records used by the inline assembly. */ +void __kfunc_btf_root(void) +{ + asm volatile ("" : + : "r"(&bpf_kfunc_test_uninit_multi), + "r"(&bpf_kfunc_test_uninit_pair), + "r"(&bpf_kfunc_test_uninit_stack)); +} + +SEC("tc") +__success __retval(42) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void multiple_outputs(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "call %[bpf_kfunc_test_uninit_multi];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u32 *)(r10 - 8);" + "if r1 != 0x2a2a2a2a goto 1f;" + "r1 = *(u32 *)(r10 - 4);" + "if r1 == 0x2a2a2a2a goto 2f;" + "1:;" + "r0 = 0;" + "2:;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_multi) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -16+0 size 4") +__naked void variable_size_preserves_other_output(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 7;" + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_kfunc_test_uninit_multi];" + "r0 = *(u32 *)(r10 - 16);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_multi) : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 4") +__naked void output_after_by_value_argument(void) +{ + asm volatile ( + "r1 = 20;" + "r2 = 22;" + "r3 = r10;" + "r3 += -8;" + "call %[bpf_kfunc_test_uninit_pair];" + "r0 = *(u32 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_pair) : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) +SEC("tc") +__arch_x86_64 __arch_arm64 __arch_riscv64 +__success __retval(15) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 4") +__naked void output_passed_on_stack(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "r6 = r10;" + "r6 += -8;" + "*(u64 *)(r11 - 8) = r6;" + "call %[bpf_kfunc_test_uninit_stack];" + "r0 = *(u32 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_stack) : __clobber_all); +} +#endif + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_large_stack.c b/tools/testing/selftests/bpf/progs/verifier_large_stack.c new file mode 100644 index 0000000..c2a3ddf --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_large_stack.c
@@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * Programs may use MAX_BPF_STACK_JIT (2 KiB) of stack on JITs that support + * large stacks, combined over a call chain, with no separate limit on a + * single frame. Interpreted programs and other JITs keep 512 bytes. + */ + +SEC("socket") +__description("single frame of 2048 bytes") +__load_if_large_stack() +__success __success_unpriv __retval(42) +__naked void single_frame_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + r0 = *(u64*)(r1 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("single frame of 2048 bytes without large stack support") +__load_if_no_large_stack() +__failure __msg("invalid write to stack R1 off=-2048 size=8") +__naked void single_frame_2048_no_large_stack(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_leaf(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 512) = r1; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_2(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_leaf; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_3(void) +{ + asm volatile (" \ + r1 = 3; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_2; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_4(void) +{ + asm volatile (" \ + r1 = 4; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_3; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("four frames of 512 bytes fit the 2 KiB budget") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void four_frames_of_512(void) +{ + asm volatile (" \ + call frame_512_depth_4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("five frames of 512 bytes exceed the 2 KiB budget") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is 2560. Too large") +__naked void five_frames_of_512(void) +{ + asm volatile (" \ + r1 = 5; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_1536_leaf(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame calling a 1536-byte frame") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void uneven_frames_fit(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_1536_leaf; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("520-byte frame calling a 1536-byte frame") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void uneven_frames_exceed(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + call frame_1536_leaf; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +#ifdef __BPF_FEATURE_MAY_GOTO +/* may_goto adds its counter below the frame; a JIT does not hold that against the budget */ +SEC("socket") +__description("frame of 2048 bytes with may_goto") +__load_if_large_stack() +__success __retval(42) +__naked void frame_2048_with_may_goto(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u32*)(r1 + 0) = r0; \ + may_goto l0_%=; \ + r2 = 100; \ + l0_%=: \ + exit; \ +" ::: __clobber_all); +} +#endif + +SEC("socket") +__description("variable offset write reaching 2048 bytes deep") +__load_if_large_stack() +__success +__naked void var_off_write_to_2048(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r2 = r10; \ + r2 += -2048; \ + r2 += r0; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__description("variable offset write reaching 2056 bytes deep") +__load_if_large_stack() +__failure __msg("invalid variable-offset write to stack R2") +__naked void var_off_write_to_2056(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r2 = r10; \ + r2 += -2056; \ + r2 += r0; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Each frame of a private stack gets the whole budget. */ +__used __naked +static void priv_stack_frame_2048(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 2048) = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("kprobe") +__description("private stack: two frames of 2048 bytes") +__load_if_large_stack() +__arch_x86_64 +__arch_arm64 +__success __log_level(4) +__msg("stack depth max 2048") +__msg("subprog 0 (private_stack_two_frames) main {{.*}} stack 2048") +__msg("subprog 1 (priv_stack_frame_2048) static {{.*}} stack 2048") +__naked void private_stack_two_frames(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 2048) = r1; \ + call priv_stack_frame_2048; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +/* + * A tail call unwinds the frame of the program doing it, so a large main + * frame is fine; the 256-byte rule only concerns the frames of callers of a + * subprog that tail calls. + */ +SEC("tc") +__description("tail call from a 1 KiB frame") +__load_if_large_stack() +__success +__naked void tail_call_from_large_frame(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 1024) = r2; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +/* + * bpf_clone_redirect() can run the program again on top of its own frame, + * ten frames deep, so a program calling it keeps the 512-byte budget. The + * redirect that happens after the program returns does not. + */ +SEC("tc") +__description("1 KiB frame with bpf_clone_redirect keeps the 512-byte budget") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-1024 size=8") +__naked void clone_redirect_keeps_512(void) +{ + asm volatile (" \ + r6 = r1; \ + r1 = r10; \ + r1 += -1024; \ + r0 = 42; \ + *(u64 *)(r1 + 0) = r0; \ + r1 = r6; \ + r2 = 1; \ + r3 = 0; \ + call %[bpf_clone_redirect]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_clone_redirect) + : __clobber_all); +} + +SEC("tc") +__description("1 KiB frame with bpf_redirect keeps the 2 KiB budget") +__load_if_large_stack() +__success +__naked void redirect_keeps_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -1024; \ + r0 = 42; \ + *(u64 *)(r1 + 0) = r0; \ + r1 = 1; \ + r2 = 0; \ + call %[bpf_redirect]; \ + exit; \ +" : + : __imm(bpf_redirect) + : __clobber_all); +} + +/* Global subprogs are verified on their own but share the call chain budget. */ +__used __naked int global_frame_1536(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame calling a 1536-byte global subprog") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void global_subprog_fits(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call global_frame_1536; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("520-byte frame calling a 1536-byte global subprog") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void global_subprog_exceeds(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + call global_frame_1536; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Callback frames are part of the chain of the helper that calls them. */ +static __naked int loop_cb_1536(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame with a 1536-byte bpf_loop callback") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void loop_callback_fits(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 1; \ + r2 = %[loop_cb_1536]; \ + r3 = 0; \ + r4 = 0; \ + call %[bpf_loop]; \ + r0 = 0; \ + exit; \ +" : + : __imm_ptr(loop_cb_1536), + __imm(bpf_loop) + : __clobber_common); +} + +SEC("socket") +__description("520-byte frame with a 1536-byte bpf_loop callback") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void loop_callback_exceeds(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + r1 = 1; \ + r2 = %[loop_cb_1536]; \ + r3 = 0; \ + r4 = 0; \ + call %[bpf_loop]; \ + r0 = 0; \ + exit; \ +" : + : __imm_ptr(loop_cb_1536), + __imm(bpf_loop) + : __clobber_common); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 4134087..49e8cd4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c
@@ -9,13 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 1); -} arena SEC(".maps"); + (__clang_major__ >= 18 || defined(__BPF_FEATURE_LDSX)) SEC("socket") __description("LDSX, S8") @@ -263,6 +257,14 @@ __naked void ldsx_ctx_8(void) : __clobber_all); } +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + SEC("syscall") __description("Arena LDSX Disasm") __success @@ -286,6 +288,19 @@ __jited("add x11, x0, x28") __jited("ldrsh x22, [x11, #0x18]") __jited("add x11, x0, x28") __jited("ldrsb x22, [x11, #0x20]") +__arch_riscv64 +__jited("add t2, t6, s7") +__jited("lw s3, 0x10(t2)") +__jited("add t2, t6, s7") +__jited("lh s3, 0x18(t2)") +__jited("add t2, t6, s7") +__jited("lb s3, 0x20(t2)") +__jited("add t2, a0, s7") +__jited("lw s4, 0x10(t2)") +__jited("add t2, a0, s7") +__jited("lh s4, 0x18(t2)") +__jited("add t2, a0, s7") +__jited("lb s4, 0x20(t2)") __naked void arena_ldsx_disasm(void *ctx) { asm volatile ( @@ -317,6 +332,7 @@ __description("Arena LDSX Exception") __success __retval(0) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_exception(void *ctx) { asm volatile ( @@ -338,6 +354,7 @@ __description("Arena LDSX, S8") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s8(void *ctx) { asm volatile ( @@ -369,6 +386,7 @@ __description("Arena LDSX, S16") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s16(void *ctx) { asm volatile ( @@ -400,6 +418,7 @@ __description("Arena LDSX, S32") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s32(void *ctx) { asm volatile ( @@ -432,10 +451,13 @@ void kfunc_root(void) bpf_arena_alloc_pages(0, 0, 0, 0, 0); } +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + #else SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 401152b..5f9b11a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c
@@ -21,8 +21,6 @@ struct { __type(value, __u64); } array_map_8b SEC(".maps"); -const char snprintf_u64_fmt[] = "%llu"; - SEC("socket") __log_level(2) __msg("0: (79) r1 = *(u64 *)(r10 -8) ; use: fp0-8") @@ -63,8 +61,12 @@ __naked void read_write_join(void) SEC("socket") __log_level(2) __msg("stack use/def subprog#0 must_write_not_same_slot (d0,cs0):") -__msg("6: (7b) *(u64 *)(r2 +0) = r0{{$}}") -__msg("Live regs before insn:") +/* + * 'r2 += r1' adds a scalar, so the offset is lost (off_cnt == 0): no def, + * but the write conservatively marks the whole frame as may_def. + */ +__msg("6: (7b) *(u64 *)(r2 +0) = r0 ; may_def: fp0-8..-{{(512|2048)}}") +__msg("Program dump") __naked void must_write_not_same_slot(void) { asm volatile ( @@ -83,6 +85,31 @@ __naked void must_write_not_same_slot(void) SEC("socket") __log_level(2) +__msg("stack use/def subprog#0 may_write_two_precise_offsets (d0,cs0):") +/* + * r1 is fp-8 or fp-16, with both offsets tracked precisely (off_cnt == 2). + * A write through it cannot set def, but marks both candidate slots as may_def. + */ +__msg("6: (7b) *(u64 *)(r1 +0) = r0 ; may_def: fp0-8 fp0-16") +__naked void may_write_two_precise_offsets(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = r10;" + "if r0 > 42 goto 1f;" + "r1 += -8;" + "goto 2f;" +"1:" + "r1 += -16;" +"2:" + "*(u64 *)(r1 + 0) = r0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__log_level(2) __msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") __msg("5: (85) call bpf_map_lookup_elem#1 ; use: fp0-8h") __naked void must_write_not_same_type(void) @@ -108,11 +135,11 @@ __naked void must_write_not_same_type(void) SEC("socket") __log_level(2) -/* Callee writes fp[0]-8: stack_use at call site has slots 0,1 live */ +/* Callee writes fp[0]-8: the def is summarized as may_def at the call site */ __msg("stack use/def subprog#0 caller_stack_write (d0,cs0):") -__msg("2: (85) call pc+1{{$}}") +__msg("2: (85) call pc+1 ; may_def: fp0-8") __msg("stack use/def subprog#1 write_first_param (d1,cs2):") -__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8") +__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8 may_def: fp0-8") __naked void caller_stack_write(void) { asm volatile ( @@ -132,6 +159,155 @@ static __used __naked void write_first_param(void) ::: __clobber_all); } +struct cnt8 { long v; }; + +__noinline int global_noop_arg(struct cnt8 *p) +{ + return 0; +} + +SEC("socket") +__log_level(2) +/* Call to a global function: the argument is described by its declared type. */ +__msg("stack use/def subprog#0 global_callee_arg (d0,cs0):") +__msg("3: (85) call pc+{{.*}}; use: fp0-8 def: fp0-8 may_def: fp0-8") +__naked void global_callee_arg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_arg;" + "exit;" + ::: __clobber_all); +} + +__noinline int global_int_arg(int x) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +__msg("stack use/def subprog#0 global_noop_int_arg (d0,cs0):") +__msg_next(" 0: (bf) r1 = r10{{$}}") +__msg_next(" 1: (07) r1 += -8{{$}}") +__msg_next(" 2: (1f) r1 -= r10{{$}}") +__msg_next(" 3: (85) call pc+1{{$}}") +__msg_next(" 4: (95) exit") +__naked void global_noop_int_arg(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r1 -= r10;" /* This is a scalar now. */ + "call global_int_arg;" /* And should not be tracked as stack effect. */ + "exit;" + ::: __clobber_all); +} + +static char global_dynptr_buf[8]; + +__noinline int global_noop_dynptr_arg(struct bpf_dynptr *p) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +/* A global dynptr argument may read/write only its 16 bytes, with no definite write. */ +__msg("stack use/def subprog#0 global_callee_dynptr_arg (d0,cs0):") +__msg(" 7: (bf) r1 = r10{{$}}") +__msg_next(" 8: (07) r1 += -32{{$}}") +__msg_next(" 9: (85) call pc+1{{ +}}; use: fp0-24 fp0-32 may_def: fp0-24 fp0-32{{$}}") +__msg_next(" 10: (95) exit{{$}}") +__naked void global_callee_dynptr_arg(void) +{ + asm volatile ( + "r1 = %[global_dynptr_buf] ll;" + "r2 = 8;" + "r3 = 0;" + "r4 = r10;" + "r4 += -32;" + "call %[bpf_dynptr_from_mem];" + "r1 = r10;" + "r1 += -32;" + "call global_noop_dynptr_arg;" + "exit;" + :: __imm_addr(global_dynptr_buf), __imm(bpf_dynptr_from_mem) + : __clobber_all); +} + +struct task_struct { int pid; }; + +__noinline int global_noop_btf_arg(struct task_struct *p __arg_untrusted) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +/* Untrusted BTF pointer arguments need no caller stack state. */ +__msg("stack use/def subprog#0 global_callee_btf_arg (d0,cs0):") +__msg_next(" 0: (7a) *(u64 *)(r10 -8) = 0{{.*}}") +__msg_next(" 1: (bf) r1 = r10{{$}}") +__msg_next(" 2: (07) r1 += -8{{$}}") +__msg_next(" 3: (85) call pc+1{{$}}") +__msg_next(" 4: (95) exit{{$}}") +__naked void global_callee_btf_arg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_btf_arg;" + "exit;" + ::: __clobber_all); +} + +/* + * Cross-frame imprecise write: imprecise_frame_writer() receives a pointer + * into the caller's frame (frame 0) but conditionally replaces it with a + * pointer into its own frame (frame 1). At the store the two are joined into + * ARG_IMPRECISE (frame is unknown, mask = {0,1}), so the offset is dropped and + * the whole of both candidate frames is conservatively marked as may_def. + */ +SEC("socket") +__log_level(2) +/* The callee's write into the caller frame is summarized at the call site. */ +__msg("stack use/def subprog#0 imprecise_frame_write (d0,cs0):") +__msg("2: (85) call pc+2 ; may_def: fp0-8..-{{(512|2048)}}") +__msg("stack use/def subprog#1 imprecise_frame_writer (d1,cs2):") +__msg("12: (7b) *(u64 *)(r1 +0) = r2 ; may_def: fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}") +__naked void imprecise_frame_write(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call imprecise_frame_writer;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void imprecise_frame_writer(void) +{ + asm volatile ( + "r6 = r1;" /* save arg: pointer into frame 0 */ + "call %[bpf_get_prandom_u32];" + "r1 = r6;" /* r1 = frame0-8 (arg) */ + "if r0 == 0 goto 1f;" + "r1 = r10;" + "r1 += -8;" /* r1 = frame1-8 (own frame) */ +"1:" + "r2 = 0;" + "*(u64 *)(r1 + 0) = r2;" /* join frame0-8 | frame1-8 -> imprecise */ + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("socket") __log_level(2) __msg("stack use/def subprog#0 caller_stack_read (d0,cs0):") @@ -246,7 +422,7 @@ static __used __naked void read_first_param2(void) SEC("socket") __flag(BPF_F_TEST_STATE_FREQ) __failure -__msg("R1 type=scalar expected=map_ptr") +__msg("Possibly NULL pointer passed to trusted R1") __naked void caller_stack_pruning_callback(void) { asm volatile ( @@ -320,7 +496,7 @@ struct { } map_array SEC(".maps"); SEC("socket") -__failure __msg("invalid read from stack R2 off=-1024 size=8") +__failure __msg("invalid read from stack R2 off=-4096 size=8") __flag(BPF_F_TEST_STATE_FREQ) __naked unsigned long caller_stack_write_tail_call(void) { @@ -331,7 +507,7 @@ __naked unsigned long caller_stack_write_tail_call(void) "if r0 != 42 goto 1f;" "goto 2f;" "1:" - "*(u64 *)(r10 - 8) = -1024;" + "*(u64 *)(r10 - 8) = -4096;" "2:" "r1 = r6;" "r2 = r10;" @@ -806,9 +982,9 @@ void __kfunc_btf_root(void) */ SEC("socket") __success __log_level(2) -__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24{{$}}") -__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24{{$}}") -__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24{{$}}") +__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24 may_def: fp0-24{{$}}") +__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24 may_def: fp0-24{{$}}") +__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24 may_def: fp0-24{{$}}") __naked void kfunc_iter_stack_liveness(void) { asm volatile ( @@ -1009,8 +1185,9 @@ __naked void four_byte_read_upper_half(void) */ SEC("socket") __log_level(2) -__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") -__msg("1: (6a) *(u16 *)(r10 -4) = 0{{$}}") +__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8 may_def: fp0-8") +/* 2-byte write only partially covers the upper half: may_def, but no def. */ +__msg("1: (6a) *(u16 *)(r10 -4) = 0 ; may_def: fp0-4h") __msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") __naked void two_byte_write_no_kill(void) { @@ -1357,9 +1534,12 @@ __naked void fp_spill_loses_precision_kills_liveness(void) */ SEC("socket") __log_level(2) -/* fp-8 live at call (callee conditionally writes → slot not killed) */ -__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8") -__msg("4: (85) call pc+2{{$}}") +/* + * fp-8 live at call: callee conditionally writes it, so the slot is not killed + * (no def), but the conditional write surfaces as may_def at the call site. + */ +__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8 may_def: fp0-8") +__msg("4: (85) call pc+2 ; may_def: fp0-8") __msg("5: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") __naked void conditional_stx_in_subprog(void) { @@ -1464,7 +1644,7 @@ static __used __naked void inner_reader(void) SEC("socket") __log_level(2) /* fp-8 NOT live at call: callee writes before reading (parent_def kills it) */ -__msg("2: .12345.... (85) call pc+") +__msg("2: .1........ (85) call pc+") __naked void callee_must_write_before_read(void) { asm volatile ( @@ -1947,15 +2127,15 @@ static __used __naked void fwd_parent_key_to_helper(void) /* * Regression for keeping later helper args after a whole-stack fallback - * on an earlier local arg. The first bpf_snprintf() arg is a local + * on an earlier local arg. The bpf_map_update_elem() key is a local * frame-derived pointer with offset-imprecise tracking (`fp1 ?`), which - * conservatively marks the whole local stack live. The fourth arg still + * conservatively marks the whole local stack live. The value arg still * forwards &parent_fp-8 and must contribute nonlocal_use[0]=0:3. */ SEC("socket") __log_level(2) __success -__msg("call bpf_snprintf{{.*}} ; use: fp1-8..-512 fp0-8") +__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-{{(512|2048)}} fp0-8") __naked void helper_arg_fallback_keeps_scanning(void) { asm volatile ( @@ -1963,32 +2143,33 @@ __naked void helper_arg_fallback_keeps_scanning(void) "*(u64 *)(r10 - 8) = r1;" "r1 = r10;" "r1 += -8;" - "call helper_snprintf_parent_after_local_fallback;" + "call helper_update_parent_after_local_fallback;" "r0 = 0;" "exit;" ::: __clobber_all); } -static __used __naked void helper_snprintf_parent_after_local_fallback(void) +static __used __naked void helper_update_parent_after_local_fallback(void) { asm volatile ( "r6 = r1;" /* save &parent_fp-8 */ "call %[bpf_get_prandom_u32];" "r0 &= 8;" - "r1 = r10;" - "r1 += -16;" - "r1 += r0;" /* local fp, offset-imprecise */ - "r2 = 8;" - "r3 = %[snprintf_u64_fmt] ll;" - "r4 = r6;" /* later arg: parent fp-8 */ - "r5 = 8;" - "call %[bpf_snprintf];" + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -16;" + "r2 += r0;" /* local fp, offset-imprecise */ + "r1 = %[array_map_8b] ll;" + "r3 = r6;" /* later arg: parent fp-8 */ + "r4 = 0;" + "call %[bpf_map_update_elem];" "r0 = 0;" "exit;" : : __imm(bpf_get_prandom_u32), - __imm(bpf_snprintf), - __imm_addr(snprintf_u64_fmt) + __imm(bpf_map_update_elem), + __imm_addr(array_map_8b) : __clobber_all); } @@ -2268,7 +2449,11 @@ static __used __naked void merge_leaf_read(void) SEC("socket") __log_level(2) __success -__msg("call bpf_loop#181 ; use: fp2-8..-512 fp1-8..-512 fp0-8..-512") +__msg("stack use/def subprog#0 bpf_loop_two_callbacks (d0,cs0):") +__msg("5: (85) call pc+{{.*}};{{.*}} may_def: fp0-8..-{{(512|2048)}}{{$}}") +__msg("8: (85) call pc+{{.*}};{{.*}} may_def: fp0-8..-{{(512|2048)}}{{$}}") +__msg("call bpf_loop#181 ; use: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}" + " may_def: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}{{$}}") __naked void bpf_loop_two_callbacks(void) { asm volatile ( @@ -2386,8 +2571,27 @@ __flag(BPF_F_TEST_STATE_FREQ) __msg("subprog#2 write_first_read_second:") __msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") __msg("18: (79) r0 = *(u64 *)(r2 +0) // r1=fp0-8 r2=fp0-16{{$}}") +/* + * The callee's write into the caller frame is summarized as may_def at the + * call sites, transitively across both frames (depth 2 -> 1 -> 0). The two + * call sites differ because the shared write_first_read_second instance + * accumulates the cross-pass union ({fp-8, fp-16}) at different points + * relative to each forwarding_rw's summarization. + */ +__msg("stack use/def subprog#0 shared_instance_must_write_overwrite (d0,cs0):") +__msg("7: (85) call pc+7 ; use: fp0-8 fp0-16 may_def: fp0-8 fp0-16") +__msg("12: (85) call pc+2 ; use: fp0-8 may_def: fp0-16") +__msg("stack use/def subprog#1 forwarding_rw (d1,cs7):") +__msg("15: (85) call pc+1 ; use: fp0-8 fp0-16 may_def: fp0-8 fp0-16") +__msg("stack use/def subprog#1 forwarding_rw (d1,cs12):") +__msg("15: (85) call pc+1 ; use: fp0-8 may_def: fp0-16") __msg("stack use/def subprog#2 write_first_read_second (d2,cs15):") -__msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") +/* + * Shared across two callsites with swapped args (r1 is fp-8 on one pass, + * fp-16 on the other): must_write intersects to empty (no def), may_write + * unions to both slots. + */ +__msg("17: (7a) *(u64 *)(r1 +0) = 42 ; may_def: fp0-8 fp0-16") __msg("18: (79) r0 = *(u64 *)(r2 +0) ; use: fp0-8 fp0-16") __naked void shared_instance_must_write_overwrite(void) { @@ -2430,6 +2634,65 @@ static __used __naked void write_first_read_second(void) } /* + * merge_instances() must preserve may_write when one pass doesn't touch a frame. + * mvs_leaf is reached through a single callsite (mvs_mid) from two outer chains, + * so it is one shared (depth,callsite) instance analyzed twice and merged: + * - chain A passes r2 = map value, mvs_leaf's doesn't touch stack; + * - chain B passes r2 = caller stack slot fp-8. + * The merge must keep the stack pass's may_write even though the map pass left + * frame 0 untouched. + */ +SEC("socket") +__success +__log_level(2) +__msg("stack use/def subprog#2 mvs_leaf (d2,cs19):") +__msg("21: (7a) *(u64 *)(r2 +0) = 42 ; may_def: fp0-8") +__naked void merge_preserves_may_write(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 24) = r1;" /* map key */ + "r1 = %[map] ll;" + "r2 = r10;" + "r2 += -24;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + /* chain A: r2 = map value (ARG_NONE, not stack) */ + "r1 = r10;" + "r1 += -16;" /* unused fp anchor */ + "r2 = r0;" + "call mvs_mid;" + /* chain B: r2 = caller stack slot fp-8 */ + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call mvs_mid;" +"1:" + "r0 = 0;" + "exit;" + : : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +static __used __naked void mvs_mid(void) +{ + asm volatile ( + "call mvs_leaf;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void mvs_leaf(void) +{ + asm volatile ( + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* * Shared must_write when (callsite, depth) instance is reused. * Main calls fwd_to_stale_wr at two sites. fwd_to_stale_wr calls * stale_wr_leaf at a single internal callsite. Both calls share @@ -2514,6 +2777,153 @@ __naked void load_acquire_dont_clear_dst(void) #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ SEC("socket") +__log_level(2) +__success +__msg("lock *(u64 *)(r10 -8) += r1{{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__naked void atomic_rmw(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + ".8byte %[atomic_add];" + ".8byte %[atomic_fetch_add];" + ".8byte %[atomic_xchg];" + "r0 = 0;" + ".8byte %[atomic_cmpxchg];" + "exit;" + : + : __imm_insn(atomic_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_fetch_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_cmpxchg, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +/* + * An 8-byte load-acquire from a spill slot restores the spilled register. + * The store through the restored pointer is a definite write to fp-8. + */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = load_acquire((u64 *)(r10 -16)){{.*}}use: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void load_acquire_fills_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + ".8byte %[load_acquire_insn];" /* r2 = load_acquire(fp-16) */ + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -16)) + : __clobber_all); +} + +/* An 8-byte store-release spills the register, same as a plain store. */ +SEC("socket") +__log_level(2) +__success +__msg("store_release((u64 *)(r10 -16), r1){{.*}}def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void store_release_spills_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + ".8byte %[store_release_insn];" /* store_release(fp-16, r1): fp-16 = &fp-8 */ + "r2 = *(u64 *)(r10 - 16);" + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -16)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +/* Stack fills through xchg, fetch_add and cmpxchg instructions. */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = atomic64_xchg((u64 *)(r10 -16), r2){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__msg("r3 = atomic64_fetch_add((u64 *)(r10 -16), r3){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r3 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -16), r0, r4){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r0 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void atomic_rmw_fills_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + "r2 = 0;" + ".8byte %[atomic_xchg];" /* r2 = xchg(fp-16, r2) */ + "*(u64 *)(r2 + 0) = 42;" + "*(u64 *)(r10 - 16) = r1;" + "r3 = 0;" + ".8byte %[atomic_fetch_add];" /* r3 = fetch_add(fp-16, r3) */ + "*(u64 *)(r3 + 0) = 42;" + "*(u64 *)(r10 - 16) = r1;" + "r0 = 0;" + "r4 = 0;" + ".8byte %[atomic_cmpxchg];" /* r0 = cmpxchg(fp-16, r0, r4) */ + "*(u64 *)(r0 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_2, -16)), + __imm_insn(atomic_fetch_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_3, -16)), + __imm_insn(atomic_cmpxchg, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_4, -16)) + : __clobber_all); +} + +/* xchg returning caller's &fp-8. */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = atomic64_xchg((u64 *)(r1 +0), r2){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}; may_def: fp0-8") +__naked void atomic_rmw_fills_parent_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + "r1 = r10;" + "r1 += -16;" + "call atomic_rmw_fills_parent_spilled_ptr_callee;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void atomic_rmw_fills_parent_spilled_ptr_callee(void) +{ + asm volatile ( + "r2 = 0;" + ".8byte %[atomic_xchg];" /* r2 = xchg(r1, r2) == caller's &fp-8 */ + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_2, 0)) + : __clobber_all); +} + +SEC("socket") __success __naked void imprecise_fill_loses_cross_frame(void) { @@ -2840,3 +3250,197 @@ static __used __naked void imprecise_dst_spill_join_sub(void) :: __imm(bpf_get_prandom_u32) : __clobber_all); } + +/* + * A store that does not fully cover a 4-byte half-slot defines nothing, so a + * narrow store at the top of the frame must not turn any slot into a "def", + * least of all every slot of the frame: the earlier data at fp-8 stays live. + */ +SEC("socket") +__log_level(2) +__msg("0: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +/* narrow stores define nothing, but they may write the half-slot they touch */ +__msg("1: (73) *(u8 *)(r10 -1) = r0 ; may_def: fp0-4h") +__msg("2: (6b) *(u16 *)(r10 -4) = r0 ; may_def: fp0-4h") +__msg("3: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +__naked void narrow_store_defines_nothing(void) +{ + asm volatile ( + "r0 = *(u64 *)(r10 - 8);" + "*(u8 *)(r10 - 1) = r0;" + "*(u16 *)(r10 - 4) = r0;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + ::: __clobber_all); +} + +/* + * The same callee instance is analyzed twice: the call sites are visited in + * postorder, so the second one goes first with a precise pointer 248 bytes + * into the main frame, and the first one then passes a pointer of unknown + * offset, which reads the whole frame. The precise pass stays within the + * first word of the masks on 64-bit, so the whole-frame pass is wider under + * every stack budget and merging the second into the first has to widen the + * masks while keeping the whole-frame read. + */ +SEC("socket") +__log_level(2) +__msg("stack use/def subprog#{{[0-9]+}} merge_read_all_callee (d2,cs{{[0-9]+}}):") +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-{{(512|2048)}}") +__naked void merge_keeps_whole_frame_read(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "*(u64 *)(r10 - 248) = r1;" + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r1 = r10;" + "r1 += -16;" + "r1 += r0;" + "call merge_read_all_mid;" + "r1 = r10;" + "r1 += -248;" + "call merge_read_all_mid;" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static __used __naked void merge_read_all_mid(void) +{ + asm volatile ( + "call merge_read_all_callee;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_read_all_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +/* + * A frame pointer spilled below fp-512 is a spill like any other: the fill + * restores its identity, and a load through it reads only the slot it names + * instead of the whole frame. + */ +SEC("socket") +__log_level(2) +__load_if_large_stack() +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8{{$}}") +__naked void spill_below_512_stays_precise(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 520) = r1;" + "r1 = *(u64 *)(r10 - 520);" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +/* Unknown-size outputs still carry a possible write, without a read or kill. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_probe_read_kernel{{.*}}; may_def: fp0-8{{$}}") +__naked void helper_output_unknown_size(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 7;" + "r0 += 1;" + "r2 = r0;" + "r1 = r10;" + "r1 += -8;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32), __imm(bpf_probe_read_kernel) + : __clobber_all); +} + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u64); +} queue_8b SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u64[2]); +} queue_16b SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_BLOOM_FILTER); + __uint(max_entries, 16); + __type(value, __u64); +} bloom_8b SEC(".maps"); + +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_pop_elem{{.*}}; def: fp0-16") +__naked void helper_map_output_known_size(void) +{ + asm volatile ( + "r1 = %[queue_8b] ll;" + "r2 = r10;" + "r2 += -16;" + "call %[bpf_map_pop_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(queue_8b), __imm(bpf_map_pop_elem) + : __clobber_all); +} + +/* The maximum map value size is not a guaranteed output extent. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_pop_elem{{.*}}; use: fp0-16 fp0-24 may_def: fp0-16 fp0-24{{$}}") +__naked void helper_map_output_merged_sizes(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "call %[bpf_get_prandom_u32];" + "r1 = %[queue_8b] ll;" + "if r0 == 0 goto 1f;" + "r1 = %[queue_16b] ll;" +"1:" + "r2 = r10;" + "r2 += -24;" + "call %[bpf_map_pop_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(queue_8b), __imm_addr(queue_16b), + __imm(bpf_get_prandom_u32), __imm(bpf_map_pop_elem) + : __clobber_all); +} + +/* Bloom-filter peek treats buffer as an input despite the helper's output annotation. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_peek_elem{{.*}}; use: fp0-8{{$}}") +__naked void helper_bloom_peek_input(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = %[bloom_8b] ll;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_map_peek_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(bloom_8b), __imm(bpf_map_peek_elem) + : __clobber_all); +}
diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index d17026d..bcc34e79 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c
@@ -267,6 +267,7 @@ __naked void load_acquire_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c new file mode 100644 index 0000000..0608e98 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c
@@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * kernel/bpf/loops.c:compute_loops_in_subprog() distinguishes between + * the following cases: + * - B: backedge -> simple loop + * - C: cross edge to non-loop node -> no-op + * - D: edge to node whose header is in DFS path -> nested loop + * - E: edge to node whose header is NOT in DFS path -> irreducible + * + * Below test cases cover the above branches in various combinations. + */ + +/* Case B: single bounded loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg(" 2 3: {{.*}} (95) exit") +__naked void loop_single(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case B: two independent loops at the same nesting level. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 4{{$}}") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 2 0 1: {{.*}} (07) r0 += 1") +__msg(" 2 1 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg(" 2 3: {{.*}} (b7) r1 = 0") +__msg(" 1 3 4: {{.*}} (07) r1 += 1") +__msg(" 1 4 4 5: {{.*}} (a5) if r1 < 0xa goto pc-2") +__msg(" 5 6: {{.*}} (95) exit") +__naked void loop_two_independent(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 10 goto 2b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case B + D: nested loops. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") /* outer loop header */ +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") /* outer loop insn */ +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") /* inner loop header */ +__msg(" 1 3 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") /* inner loop insn */ +__msg(" 1 1 4 5: {{.*}} (a5) if r0 < 0xa goto pc-5") /* outer loop insn */ +__msg(" 5 6: {{.*}} (95) exit") +__naked void loop_nested(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case C: forward edges, no loops. */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg(" -1 0: {{.*}} (85) call bpf_get_prandom_u32") +__msg(" 0 1: {{.*}} (25) if r0 > 0x0 goto pc+2") +__msg(" 1 2: {{.*}} (b7) r0 = 2") +__msg(" 2 3: {{.*}} (05) goto pc+1") +__msg(" 1 4: {{.*}} (b7) r0 = 3") +__msg(" 1 5: {{.*}} (95) exit") +__naked void fwd_edges_no_loop(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r0 = 2; \ + goto 2f; \ +1: r0 = 3; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Case B + D: two sibling inner loops within one outer loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("loop at 6, nested in 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") +__msg(" 1 3 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") +__msg(" 1 1 4 5: {{.*}} (b7) r2 = 0") +__msg(" 1 1 5 6: {{.*}} (07) r2 += 1") +__msg(" 1 6 6 7: {{.*}} (a5) if r2 < 0x5 goto pc-2") +__msg(" 1 1 7 8: {{.*}} (a5) if r0 < 0xa goto pc-8") +__msg(" 8 9: {{.*}} (95) exit") +__naked void loop_nested_siblings(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 5 goto 3b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Three levels of nesting. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("loop at 5, nested in 3") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") +__msg(" 1 3 3 4: {{.*}} (b7) r2 = 0") +__msg(" 1 3 4 5: {{.*}} (07) r2 += 1") +__msg(" 1 5 5 6: {{.*}} (a5) if r2 < 0x3 goto pc-2") +__msg(" 1 3 6 7: {{.*}} (a5) if r1 < 0x5 goto pc-5") +__msg(" 1 1 7 8: {{.*}} (a5) if r0 < 0xa goto pc-8") +__msg(" 8 9: {{.*}} (95) exit") +__naked void loop_three_levels(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 3 goto 3b; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Loop with an if-else body (forward branch inside loop, Case D). */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (bf) r1 = r0") +__msg(" 1 1 2 3: {{.*}} (25) if r1 > 0x5 goto pc+1") +__msg(" 1 1 3 4: {{.*}} (b7) r1 = 1") +__msg(" 1 1 3 5: {{.*}} (0f) r0 += r1") +__msg(" 1 1 5 6: {{.*}} (a5) if r0 < 0x64 goto pc-6") +__msg(" 6 7: {{.*}} (95) exit") +__naked void loop_with_if_else(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = r0; \ + if r1 > 5 goto 2f; \ + r1 = 1; \ +2: r0 += r1; \ + if r0 < 100 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case E: irreducible loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 3, irreducible") +__msg("Program dump") +__msg(" -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg(" 0 1: {{.*}} (b7) r1 = 0") +__msg(" 1 2: {{.*}} (25) if r0 > 0x5 goto pc+2") +__msg(" 1 2 3: {{.*}} (b7) r1 = 1") +__msg(" 1 3 3 4: {{.*}} (05) goto pc+1") +__msg(" 2 5: {{.*}} (b7) r1 = 2") +__msg(" 1 3 2 6: {{.*}} (0f) r0 += r1") +__msg(" 1 3 6 7: {{.*}} (a5) if r0 < 0x10 goto pc-5") +__msg(" 7 8: {{.*}} (95) exit") +__naked void loop_irreducible(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ + if r0 > 5 goto 2f; \ +1: r1 = 1; \ + goto 3f; \ +2: r1 = 2; \ +3: r0 += r1; \ + if r0 < 16 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__failure +__log_level(2) +__msg("loop at 1") +__msg("infinite loop detected at insn 1") +__naked void loop_self(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ +1: if r0 < 10 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loops at 5 and 7 are nested inside loops at 3 and 1. + * The edge 6 -> 7 exits loop at 5 and enters loop at 7. + * Check that it is logged only as an exit from loop at 5. + * + * 0: r0 = 0; + * do { + * 1: r0++; + * 2: r1 = 0; + * do { + * 3: r1++; + * 4: r2 = 0; + * do { + * 5: r2++; + * 6: } while (r2 < 3); + * do { + * 7: r2--; + * 8: } while (r2 != 0); + * 9: } while (r1 < 4); + * 10: } while (r0 < 5); + * 11: return r0; + */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg_next(" backedge from 10, latch at 10") +__msg_next(" exit from 10 to 11") +__msg_next("loop at 3, nested in 1") +__msg_next(" backedge from 9, latch at 9") +__msg_next(" exit from 9 to 10") +__msg_next("loop at 5, nested in 3") +__msg_next(" backedge from 6, latch at 6") +__msg_next(" exit from 6 to 7") +__msg_next("loop at 7, nested in 3") +__msg_next(" backedge from 8, latch at 8") +__msg_next(" exit from 8 to 9") +__naked void loop_nested_siblings_common_ancestors(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 3 goto 3b; \ +4: r2 += -1; \ + if r2 != 0 goto 4b; \ + if r1 < 4 goto 2b; \ + if r0 < 5 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The outer loop backedge (4 -> 2) originates in an inner loop. + * For now, latch should not be identified in such cases. + */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 2{{$}}") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 5 to 7") +__msg_next("loop at 3, nested in 2") +__msg_next(" backedge from 6, latch at 5") +__msg_next(" exit from 4 to 2") +__msg_next(" exit from 5 to 7") +__naked void loop_outer_backedge_from_inner(void) +{ + asm volatile (" \ + r6 = 99; \ + r2 = 2; \ +2: r6 += 1; /* loop at 2 */ \ +3: r2 += 1; /* loop at 3 */ \ +4: if r2 == 3 goto 2b; \ +5: if r6 > 100 goto 7f; \ + goto 3b; \ +7: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm.c b/tools/testing/selftests/bpf/progs/verifier_lsm.c index c724bf3..8fbf786 100644 --- a/tools/testing/selftests/bpf/progs/verifier_lsm.c +++ b/tools/testing/selftests/bpf/progs/verifier_lsm.c
@@ -212,4 +212,17 @@ __naked int retval_load_resets_bounds(void *ctx) ::: __clobber_all); } +SEC("lsm/file_mprotect") +__description("lsm ctx is read-only") +__failure __msg("invalid bpf_context access") +__naked int reject_ctx_write(void *ctx) +{ + asm volatile ( + "r6 = 0;" + "*(u64 *)(r1 + 0) = r6;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c b/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c new file mode 100644 index 0000000..b706bf1 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c
@@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +const char xattr_zone[] = "security.bpf.zone"; +char value_buf[8] = "z"; +int scratch_count; + +SEC("lsm/inode_init_security") +__failure __msg("cannot write into rdonly_trusted_mem") +int BPF_PROG(reject_count_write, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + *xattr_count = 0; + return 0; +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_count_read, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + scratch_count = *xattr_count; + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(reject_unchecked_xattrs, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("invalid mem access 'trusted_ptr_or_null_'") +int BPF_PROG(reject_unchecked_qstr, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + scratch_count = qstr->len; + return 0; +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_spilled_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + int *saved = xattr_count; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, saved, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_forged_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, &scratch_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_stack_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + int local = 0; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, &local, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_other_ctx_arg, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, (int *)xattrs, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(reject_null_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, NULL, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("dereference of modified rdonly_trusted_mem ptr") +int BPF_PROG(reject_shifted_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count + 1, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("variable rdonly_trusted_mem access var_off=") +int BPF_PROG(reject_var_shifted_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count + (scratch_count & 1), + xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("invalid bpf_context access") +int reject_ctx_forged_count(unsigned long long *ctx) +{ + volatile unsigned long long *slot = ctx; + struct bpf_dynptr value; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + slot[4] = 0; + bpf_inode_init_xattr((struct xattr *)(long)slot[3], + (int *)(long)slot[4], xattr_zone, &value); + return 0; +} + +static __noinline int claim_via_subprog(struct xattr *xattrs, int *xattr_count, + struct bpf_dynptr *value) +{ + return bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, value); +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_count_via_subprog, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + claim_via_subprog(xattrs, xattr_count, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("access beyond struct xattr at off 24") +int BPF_PROG(reject_shifted_xattrs, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs + 1, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm.s/inode_init_security") +__failure __msg("bpf_lsm_inode_init_security is not sleepable") +int BPF_PROG(reject_sleepable, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + return 0; +} + +SEC("lsm_cgroup/inode_init_security") +__failure __msg("calling kernel function bpf_inode_init_xattr is not allowed") +int BPF_PROG(reject_lsm_cgroup, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_setxattr") +__failure __msg("calling kernel function bpf_inode_init_xattr is not allowed") +int BPF_PROG(reject_wrong_hook, struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, const void *value, size_t size, int flags) +{ + struct bpf_dynptr val; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &val); + bpf_inode_init_xattr(NULL, &scratch_count, xattr_zone, &val); + return 0; +}
diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index d3be69a..621248a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c
@@ -154,8 +154,7 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") -__msg("map_ptr_or_null, but this argument accepts map_ptr") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void on_the_inner_map_pointer(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c index c01abf5..4b1eadd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c
@@ -58,7 +58,7 @@ int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) } SEC("?tc") -__failure __msg("type=map_ptr_or_null expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) { char th[sizeof(struct tcphdr)] = {};
diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index db7e30d..faf9eec 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c
@@ -173,4 +173,125 @@ __naked void timed_may_goto_preserves_regs(void) : __clobber_all); } +__used __naked static void mg_ret1(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_ret2(void) +{ + asm volatile ( + "r0 = 2;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_call_first(void) +{ + asm volatile ( + "call mg_ret1;" + "r1 = 0;" +"1:" + ".8byte %[may_goto];" + "r1 += 1;" + "if r1 < 10 goto 1b;" + "exit;" + : + : __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) + : __clobber_all); +} + +/* + * mg_ret1 and mg_ret2 are placed in front of mg_call_first, + * so the offset of its call is negative. + */ +SEC("socket") +__description("may_goto in subprog that starts with a call") +__success +__retval(1) +__naked void may_goto_subprog_call_first(void) +{ + asm volatile ( + "call mg_ret1;" + "call mg_ret2;" + "call mg_call_first;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_cb_stop(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_cb_cont(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* r1 is zero, but not a constant, so that bpf_loop() is not inlined */ +__used __naked static void mg_func_first(void) +{ + asm volatile ( + "r2 = %[mg_cb_stop] ll;" + "r4 = r1;" + "r1 = 2;" + "r3 = 0;" + "call %[bpf_loop];" + "r1 = 0;" +"1:" + ".8byte %[may_goto];" + "r1 += 1;" + "if r1 < 10 goto 1b;" + "exit;" + : + : __imm_addr(mg_cb_stop), + __imm(bpf_loop), + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) + : __clobber_all); +} + +/* + * mg_cb_stop stops bpf_loop() after the first iteration, + * mg_cb_cont lets it do both. + */ +SEC("socket") +__description("may_goto in subprog that starts with ld_imm64 of a func") +__success +__retval(1) +__naked void may_goto_subprog_func_first(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = %[mg_cb_stop] ll;" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = %[mg_cb_cont] ll;" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "call %[bpf_ktime_get_ns];" + "r1 = r0;" + "r1 >>= 63;" + "call mg_func_first;" + "exit;" + : + : __imm_addr(mg_cb_stop), + __imm_addr(mg_cb_cont), + __imm(bpf_loop), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index a4d8814..195b27a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c
@@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_MOVSX)) SEC("socket") __description("MOV32SX, S8") @@ -343,6 +343,7 @@ label_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_mtu.c b/tools/testing/selftests/bpf/progs/verifier_mtu.c index 256956e..2f71f70 100644 --- a/tools/testing/selftests/bpf/progs/verifier_mtu.c +++ b/tools/testing/selftests/bpf/progs/verifier_mtu.c
@@ -17,4 +17,92 @@ int tc_uninit_mtu(struct __sk_buff *ctx) return TCX_PASS; } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_WRONLY_PROG); + __type(key, __u32); + __type(value, __u32); +} map_mtu_wo SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} map_mtu_rw SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_RDONLY_PROG); + __type(key, __u32); + __type(value, __u32); +} map_mtu_ro SEC(".maps"); + +SEC("tc/ingress") +__failure __msg("read from map forbidden") +int tc_writeonly_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("tc/ingress") +__success +int tc_readwrite_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_rw, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("xdp") +__failure __msg("read from map forbidden") +int xdp_writeonly_mtu(struct xdp_md *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return XDP_PASS; +} + +SEC("tc/ingress") +__failure __msg("write into map forbidden") +int tc_readonly_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_ro, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("tc/ingress") +__success +int tc_writeonly_output(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *out; + + out = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (out) + bpf_get_current_comm(out, sizeof(*out)); + return TCX_PASS; +} + char LICENSE[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c index 967f4e6..fd418f2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c
@@ -61,6 +61,7 @@ __naked void percpu_addr(void) SEC("raw_tp") __description("percpu addr dummy") +__skip("percpu addr tests need x86") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index ea0a7e7..3d9994a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c
@@ -377,6 +377,7 @@ int private_stack_max_depth(void) SEC("kprobe") __description("private stack is not supported, use a dummy test") +__skip("private stack is not supported") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c index c689665..0fe631411 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c
@@ -84,6 +84,8 @@ __naked void skb_load_bytes_zero_len(void) SEC("tc") __description("raw_stack: skb_load_bytes, no init") __success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 8") __naked void skb_load_bytes_no_init(void) { asm volatile (" \ @@ -103,6 +105,8 @@ __naked void skb_load_bytes_no_init(void) SEC("tc") __description("raw_stack: skb_load_bytes, init") __success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv __naked void stack_skb_load_bytes_init(void) { asm volatile (" \ @@ -236,6 +240,7 @@ __naked void load_bytes_spilled_regs_data(void) SEC("tc") __description("raw_stack: skb_load_bytes, invalid access 1") +__load_if_no_large_stack() __failure __msg("invalid write to stack R3 off=-513 size=8") __naked void load_bytes_invalid_access_1(void) { @@ -254,6 +259,26 @@ __naked void load_bytes_invalid_access_1(void) } SEC("tc") +__description("raw_stack: skb_load_bytes, invalid access 1, large stack") +__load_if_large_stack() +__failure __msg("invalid write to stack R3 off=-2049 size=8") +__naked void load_bytes_invalid_access_1_large(void) +{ + asm volatile (" \ + r2 = 4; \ + r6 = r10; \ + r6 += -2049; \ + r3 = r6; \ + r4 = 8; \ + call %[bpf_skb_load_bytes]; \ + r0 = *(u64*)(r6 + 0); \ + exit; \ +" : + : __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +SEC("tc") __description("raw_stack: skb_load_bytes, invalid access 2") __failure __msg("invalid write to stack R3 off=-1 size=8") __naked void load_bytes_invalid_access_2(void)
diff --git a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c index 199ad18..799db6f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +++ b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c
@@ -344,7 +344,7 @@ __naked void potential_reference_to_system_key(void) SEC("tc") __description("reference tracking: release reference without check") -__failure __msg("type=sock_or_null expected=sock") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void tracking_release_reference_without_check(void) { asm volatile ( @@ -363,7 +363,7 @@ __naked void tracking_release_reference_without_check(void) SEC("tc") __description("reference tracking: release reference to sock_common without check") -__failure __msg("type=sock_common_or_null expected=sock") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void to_sock_common_without_check(void) { asm volatile ( @@ -1288,7 +1288,7 @@ l1_%=: r1 = r6; \ SEC("tc") __description("reference tracking: bpf_sk_release(listen_sk)") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_listen_sk(void) { asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c new file mode 100644 index 0000000..661ce9c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c
@@ -0,0 +1,3451 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <stdbool.h> +#include <stdint.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +#define COND_TEST(name, _initial, _step, _bound, msg, body) \ +SEC("xdp") \ +__success \ +__log_level(2) \ +__flag(BPF_F_TEST_STATE_FREQ) \ +__msg("loop header at {{[0-9]+}}, " msg) \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r6 = %[initial] ll;" \ + "r7 = %[bound] ll;" \ + "r9 = 0;" \ + body \ + "2: r0 = 0;" \ + "exit;" \ + : \ + : __imm_const(initial, _initial), \ + __imm_const(step, _step), \ + __imm_const(bound, _bound) \ + : __clobber_all); \ +} + +/* Optional assembly runs before the tested condition, preserving its latch. */ +#define PRE__COND_TEST(name, op, initial, step, bound, msg, ...) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" __VA_ARGS__ \ + "if r6 " op " r7 goto 2f;" \ + "r6 += r8;" \ + "goto 1b;") + +#define POST_COND_TEST(name, op, initial, step, bound, msg, ...) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" __VA_ARGS__ \ + "r6 += r8;" \ + "if r6 " op " r7 goto 1b;") + +/* + * Bound fallback verification of otherwise infinite or extremely long loops. + * JSET is not a linear condition, SCEV ignores it and the tested condition + * alone decides the iteration count. + */ +#define LIMIT_ITERATIONS "r9 += 1; if r9 & 8 goto 2f;" + +#define PRE__COND_TEST2(name, op, initial, step, bound, msg) \ + PRE__COND_TEST(name, op, initial, step, bound, msg, LIMIT_ITERATIONS) + +#define POST_COND_TEST32(name, op, initial, step, bound, msg) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" LIMIT_ITERATIONS \ + "w6 += w8;" \ + "if w6 " op " w7 goto 1b;") + +#define ITER_U64_MAX (~0ULL) +#define ITER_S64_MAX ((1ULL << 63) - 1) +#define ITER_S64_MIN (1ULL << 63) +#define ITER_U32_MAX ((1ULL << 32) - 1) + +struct map_val { + char foo[1024]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct map_val); +} map SEC(".maps"); + +typeof(map) other_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, __u64); +} small_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); + __array(values, struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(map_flags, BPF_F_INNER_MAP); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct map_val); + }); +} outer_map SEC(".maps"); + +struct bpf_iter__bpf_map_elem { + struct bpf_iter_meta *meta; + struct bpf_map *map; + void *key; + void *value; +}; + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__naked void simple_loop1(void) +{ + asm volatile (" \ + r0 = 0; \ +loop_%=: \ + if r0 == 10 goto exit_%=; \ + r0 += 1; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("8: (7b) *(u64 *)(r1 +0) = r0 ; *fp-8 (+ *fp-8 1) -> ?") +__naked void indirect_write_invalidates_scev(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ +loop_%=: \ + r0 = *(u64 *)(r10 - 8); \ + if r0 == 10 goto exit_%=; \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + r1 = r10; \ + r1 += -8; \ + *(u64 *)(r1 + 0) = r0; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Even a no-op global callee can be freplaced with one that writes to *p. */ +__noinline int global_noop_counter(__u64 *p) +{ + return 0; +} + +SEC("xdp") +__success +__log_level(2) +__msg("8: (85) call pc+{{[0-9]+}} {{.*}}; *fp-8 (+ *fp-8 1) -> ?") +__naked void global_call_invalidates_scev(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 8) = r0;" + "r6 = 0;" +"1:" + "r0 = *(u64 *)(r10 - 8);" + "r0 += 1;" + "*(u64 *)(r10 - 8) = r0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_counter;" + "r6 += 1;" + "if r6 < 3 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg_next(" scev at latch 3:") +__msg_next(" r0=*fp-8 / (linear *fp-8 1)") +__msg_next(" *fp-8=*fp-8 / (linear *fp-8 1)") +__naked void simple_loop2(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ +loop_%=: \ + r0 = *(u64 *)(r10 - 8); \ + if r0 == 10 goto exit_%=; \ + r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 2:") +__msg_next(" r1=(+ r1 1) / (linear r1 1)") +__naked void meet_agrees(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ +1: \ + if r1 == 2 goto 3f; \ + if r0 == 7 goto 2f; \ + r1 += 1; \ + goto 1b; \ +2: \ + r1 += 1; \ + goto 1b; \ +3: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__failure +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__naked void meet_disagrees(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 3f; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 7 goto 1b; \ + r6 += 1; \ + goto 1b; \ +3: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Both entry state values for r1 should be accounted for */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 2:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=42 / (any r1 42)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=42 / (any r1 42)") +__msg("loop header at 2, widening r0 to 0..10 step 1") +__msg_next("loop header at 2, widening r1 to 7..42 step 1") +__naked void entry_state_join(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = 7; \ +1: if r0 == 10 goto 2f; /* r1 in [7,42] */ \ + r0 += 1; \ + r1 = 42; \ + goto 1b; \ +2: r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(bswap32 (bswap32 (zext32 (- (- (zext32 (+ (>>...) 1))))))) / ?") +__naked void expr_chain(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 2f; \ + r6 += 1; \ + r6 <<= 32; \ + r6 >>= 32; \ + w6 += 1; \ + r6 = -r6; \ + w6 = -w6; \ + r6 = bswap32 r6; \ + r6 = bswap32 r6; \ + goto 1b; \ +2: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#define ALU_OP(insn) "r1 = r6; " insn "; r0 += r1;" + +SEC("xdp") +__success +__log_level(2) +__msg("r1 += 2 {{.*}}; r1 r6 -> (+ r6 2)") +__msg("r1 += r7 {{.*}}; r1 r6 -> (+ r6 r7)") +__msg("w1 += 2 {{.*}}; r1 r6 -> (zext32 (+ r6 2))") +__msg("w1 += w7 {{.*}}; r1 r6 -> (zext32 (+ r6 r7))") +__msg("r1 -= 2 {{.*}}; r1 r6 -> (- r6 2)") +__msg("r1 -= r7 {{.*}}; r1 r6 -> (- r6 r7)") +__msg("w1 -= 2 {{.*}}; r1 r6 -> (zext32 (- r6 2))") +__msg("w1 -= w7 {{.*}}; r1 r6 -> (zext32 (- r6 r7))") +__msg("r1 *= 2 {{.*}}; r1 r6 -> (* r6 2)") +__msg("r1 *= r7 {{.*}}; r1 r6 -> (* r6 r7)") +__msg("w1 *= 2 {{.*}}; r1 r6 -> (zext32 (* r6 2))") +__msg("w1 *= w7 {{.*}}; r1 r6 -> (zext32 (* r6 r7))") +__msg("r1 /= 2 {{.*}}; r1 r6 -> (/ r6 2)") +__msg("r1 /= r7 {{.*}}; r1 r6 -> (/ r6 r7)") +__msg("w1 /= 2 {{.*}}; r1 r6 -> (zext32 (/ r6 2))") +__msg("w1 /= w7 {{.*}}; r1 r6 -> (zext32 (/ r6 r7))") +__msg("r1 s/= 2 {{.*}}; r1 r6 -> (s/ r6 2)") +__msg("r1 s/= r7 {{.*}}; r1 r6 -> (s/ r6 r7)") +__msg("w1 s/= 2 {{.*}}; r1 r6 -> (zext32 (s/ r6 2))") +__msg("w1 s/= w7 {{.*}}; r1 r6 -> (zext32 (s/ r6 r7))") +__msg("r1 %= 2 {{.*}}; r1 r6 -> (% r6 2)") +__msg("r1 %= r7 {{.*}}; r1 r6 -> (% r6 r7)") +__msg("w1 %= 2 {{.*}}; r1 r6 -> (zext32 (% r6 2))") +__msg("w1 %= w7 {{.*}}; r1 r6 -> (zext32 (% r6 r7))") +__msg("r1 s%= 2 {{.*}}; r1 r6 -> (s% r6 2)") +__msg("r1 s%= r7 {{.*}}; r1 r6 -> (s% r6 r7)") +__msg("w1 s%= 2 {{.*}}; r1 r6 -> (zext32 (s% r6 2))") +__msg("w1 s%= w7 {{.*}}; r1 r6 -> (zext32 (s% r6 r7))") +__msg("r1 |= 2 {{.*}}; r1 r6 -> (| r6 2)") +__msg("r1 |= r7 {{.*}}; r1 r6 -> (| r6 r7)") +__msg("w1 |= 2 {{.*}}; r1 r6 -> (zext32 (| r6 2))") +__msg("w1 |= w7 {{.*}}; r1 r6 -> (zext32 (| r6 r7))") +__msg("r1 &= 2 {{.*}}; r1 r6 -> (& r6 2)") +__msg("r1 &= r7 {{.*}}; r1 r6 -> (& r6 r7)") +__msg("w1 &= 2 {{.*}}; r1 r6 -> (zext32 (& r6 2))") +__msg("w1 &= w7 {{.*}}; r1 r6 -> (zext32 (& r6 r7))") +__msg("r1 ^= 2 {{.*}}; r1 r6 -> (^ r6 2)") +__msg("r1 ^= r7 {{.*}}; r1 r6 -> (^ r6 r7)") +__msg("w1 ^= 2 {{.*}}; r1 r6 -> (zext32 (^ r6 2))") +__msg("w1 ^= w7 {{.*}}; r1 r6 -> (zext32 (^ r6 r7))") +__msg("r1 <<= 2 {{.*}}; r1 r6 -> (<< r6 2)") +__msg("r1 <<= r7 {{.*}}; r1 r6 -> (<< r6 r7)") +__msg("w1 <<= 2 {{.*}}; r1 r6 -> (zext32 (<< r6 2))") +__msg("w1 <<= w7 {{.*}}; r1 r6 -> (zext32 (<< r6 r7))") +__msg("r1 >>= 2 {{.*}}; r1 r6 -> (>> r6 2)") +__msg("r1 >>= r7 {{.*}}; r1 r6 -> (>> r6 r7)") +__msg("w1 >>= 2 {{.*}}; r1 r6 -> (zext32 (>> r6 2))") +__msg("w1 >>= w7 {{.*}}; r1 r6 -> (zext32 (>> r6 r7))") +__msg("r1 s>>= 2 {{.*}}; r1 r6 -> (s>> r6 2)") +__msg("r1 s>>= r7 {{.*}}; r1 r6 -> (s>> r6 r7)") +__msg("w1 s>>= 2 {{.*}}; r1 r6 -> (zext32 (s>> r6 2))") +__msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> r6 r7))") +__msg("r1 = -r1 {{.*}}; r1 r6 -> (- r6)") +__msg("w1 = -w1 {{.*}}; r1 r6 -> (zext32 (- r6))") +__msg("r1 = 42 {{.*}}; r1 r6 -> 42") +__msg("w1 = 42 {{.*}}; r1 r6 -> (zext32 42)") +__msg("r1 = r7 {{.*}}; r1 r6 -> r7") +__msg("w1 = w7 {{.*}}; r1 r6 -> (zext32 r7)") +__msg("r1 = (s8)r6 {{.*}}; r1 r6 -> (sext8 r6)") +__msg("r1 = (s16)r6 {{.*}}; r1 r6 -> (sext16 r6)") +__msg("r1 = (s32)r6 {{.*}}; r1 r6 -> (sext32 r6)") +__msg("w1 = (s8)w6 {{.*}}; r1 r6 -> (zext32 (sext8 r6))") +__msg("w1 = (s16)w6 {{.*}}; r1 r6 -> (zext32 (sext16 r6))") +__msg("r1 = bswap16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = bswap32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = bswap64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (zext16 r6)") +__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (zext32 r6)") +__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)") +__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = be64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +#else +__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = le64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (zext16 r6)") +__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (zext32 r6)") +__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)") +#endif +__naked void alu_ops_exprs(void) +{ + asm volatile ( + "r6 = 7;" + "r7 = 2;" + "r8 = 0;" +"1:" + "r0 = 0;" + ALU_OP("r1 += 2") + ALU_OP("r1 += r7") + ALU_OP("w1 += 2") + ALU_OP("w1 += w7") + ALU_OP("r1 -= 2") + ALU_OP("r1 -= r7") + ALU_OP("w1 -= 2") + ALU_OP("w1 -= w7") + ALU_OP("r1 *= 2") + ALU_OP("r1 *= r7") + ALU_OP("w1 *= 2") + ALU_OP("w1 *= w7") + ALU_OP("r1 /= 2") + ALU_OP("r1 /= r7") + ALU_OP("w1 /= 2") + ALU_OP("w1 /= w7") + ALU_OP("r1 s/= 2") + ALU_OP("r1 s/= r7") + ALU_OP("w1 s/= 2") + ALU_OP("w1 s/= w7") + ALU_OP("r1 %%= 2") + ALU_OP("r1 %%= r7") + ALU_OP("w1 %%= 2") + ALU_OP("w1 %%= w7") + ALU_OP("r1 s%%= 2") + ALU_OP("r1 s%%= r7") + ALU_OP("w1 s%%= 2") + ALU_OP("w1 s%%= w7") + ALU_OP("r1 |= 2") + ALU_OP("r1 |= r7") + ALU_OP("w1 |= 2") + ALU_OP("w1 |= w7") + ALU_OP("r1 &= 2") + ALU_OP("r1 &= r7") + ALU_OP("w1 &= 2") + ALU_OP("w1 &= w7") + ALU_OP("r1 ^= 2") + ALU_OP("r1 ^= r7") + ALU_OP("w1 ^= 2") + ALU_OP("w1 ^= w7") + ALU_OP("r1 <<= 2") + ALU_OP("r1 <<= r7") + ALU_OP("w1 <<= 2") + ALU_OP("w1 <<= w7") + ALU_OP("r1 >>= 2") + ALU_OP("r1 >>= r7") + ALU_OP("w1 >>= 2") + ALU_OP("w1 >>= w7") + ALU_OP("r1 s>>= 2") + ALU_OP("r1 s>>= r7") + ALU_OP("w1 s>>= 2") + ALU_OP("w1 s>>= w7") + ALU_OP("r1 = -r1") + ALU_OP("w1 = -w1") + ALU_OP("r1 = 42") + ALU_OP("w1 = 42") + ALU_OP("r1 = r7") + ALU_OP("w1 = w7") + ALU_OP("r1 = (s8)r6") + ALU_OP("r1 = (s16)r6") + ALU_OP("r1 = (s32)r6") + ALU_OP("w1 = (s8)w6") + ALU_OP("w1 = (s16)w6") + ALU_OP("r1 = bswap16 r1") + ALU_OP("r1 = bswap32 r1") + ALU_OP("r1 = bswap64 r1") + ALU_OP("r1 = le16 r1") + ALU_OP("r1 = le32 r1") + /* + * Expressions unchanged by transfer() do not appear in the log. + * Add r1 += 1 to touch the expression and make it appear in the log. + */ + ALU_OP("r1 = le64 r1; r1 += 1") + ALU_OP("r1 = be16 r1") + ALU_OP("r1 = be32 r1") + ALU_OP("r1 = be64 r1; r1 += 1") + "r8 += 1;" + "if r8 < 2 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +#undef ALU_OP + +/* Reset operands and consume results to keep transfer changes live. */ +#define ST_OP(stmt) \ + "r0 = 0; r1 = 1; r2 = 2;" \ + "*(u64 *)(r10 - 8) = -8;" \ + stmt ";" \ + "r0 += r1; r0 += r2;" \ + "r3 = *(u64 *)(r10 - 8); r0 += r3;" + +SEC("xdp") +__success +__log_level(2) +__msg("*(u64 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> 1") +__msg("*(u32 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill32 1)") +__msg("*(u16 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill16 1)") +__msg("*(u8 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill8 1)") +__msg("r1 = *(u64 *)(r10 -8) {{.*}}; r1 1 -> -8") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> (zext32 -8)") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (zext16 -8)") +__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> (zext8 -8)") +__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> (sext32 -8)") +__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> (sext16 -8)") +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> (sext8 -8)") +__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> (zext8 -8)") +__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> (zext16 -8)") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> (zext32 -8)") +#else +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> _") +#endif +__msg("r2 = load_acquire((u64 *)(r10 -8)) {{.*}}; r2 2 -> -8") +__msg("store_release((u8 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill8 1)") +__msg("store_release((u16 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill16 1)") +__msg("store_release((u32 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill32 1)") +__msg("store_release((u64 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> 1") +__msg("lock *(u64 *)(r10 -8) += r1 {{.*}}; *fp-8 -8 -> _") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> _, *fp-8 -8 -> _") +__msg("*(u64 *)(r10 -8) = 42 {{.*}}; *fp-8 -8 -> 42") +__msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> _") +__naked void store_load_exprs(void) +{ + asm volatile ( + "r7 = 0;" +"1:" + ST_OP("*(u64 *)(r10 - 8) = r1") + ST_OP("*(u32 *)(r10 - 8) = r1") + ST_OP("*(u16 *)(r10 - 8) = r1") + ST_OP("*(u8 *)(r10 - 8) = r1") + + ST_OP("r1 = *(u64 *)(r10 - 8)") + ST_OP("r1 = *(u32 *)(r10 - 8)") + ST_OP("r1 = *(u16 *)(r10 - 8)") + ST_OP("r1 = *(u8 *)(r10 - 8)") + ST_OP("r1 = *(s32 *)(r10 - 8)") + ST_OP("r1 = *(s16 *)(r10 - 8)") + ST_OP("r1 = *(s8 *)(r10 - 8)") + + ST_OP(".8byte %[load_acquire8]") + ST_OP(".8byte %[load_acquire16]") + ST_OP(".8byte %[load_acquire32]") + ST_OP(".8byte %[load_acquire64]") + ST_OP(".8byte %[store_release8]") + ST_OP(".8byte %[store_release16]") + ST_OP(".8byte %[store_release32]") + ST_OP(".8byte %[store_release64]") + + ST_OP(".8byte %[atomic_add64]") + ST_OP(".8byte %[atomic_fetch_add64]") + ST_OP(".8byte %[atomic_fetch_add32]") + ST_OP(".8byte %[atomic_xchg64]") + ST_OP(".8byte %[atomic_cmpxchg64]") + + ST_OP("*(u64 *)(r10 - 8) = 42") + ST_OP("*(u8 *)(r10 - 7) = r0") + + "r7 += 1;" + "if r7 < 2 goto 1b;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_add64, BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_xchg64, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(load_acquire8, BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire16, BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire32, BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire64, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)), + __imm_insn(store_release8, BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release16, BPF_ATOMIC_OP(BPF_H, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release32, BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release64, BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_cmpxchg64, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8)), + __imm_insn(atomic_fetch_add32, BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_fetch_add64, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +#undef ST_OP + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next(" scev at latch 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next("scev at header 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next(" scev at latch 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__naked void nested_loop1(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 2f; \ + r6 += 1; \ + r7 = 0; \ +3: \ + if r7 == 2 goto 4f; \ + r7 += 1; \ + goto 3b; \ +4: \ + goto 1b; \ +2: \ + r0 = r7; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__msg("loop at 1") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 1 to 7") +__msg_next("loop at 4, nested in 1") +__msg_next(" backedge from 6, latch at 4") +__msg_next(" exit from 4 to 1") +__msg("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next("scev at header 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next(" scev at latch 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("loop header at 1, latch not identified") +__msg("loop header at 4, widening r7 to 0..2 step 1") +__log_level(2) +__naked void nested_loop_hdr_backedge1(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 3f; \ + r6 += 1; \ + r7 = 0; \ +2: \ + if r7 == 2 goto 1b; \ + r7 += 1; \ + goto 2b; \ +3: \ + r0 = r7; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Outer loop {2, 3, 4, 5} contains inner loop {3, 4, 5}. + * The outer loop's only backedge is 4 -> 2, and its latch is also at 4, + * inside the inner loop. Outer loop's latch index is not computed in + * such a case, as there is no mechanism to build SCEVs for such latches + * at the moment. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 2{{$}}") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 5 to 6") +__msg_next("loop at 3, nested in 2") +__msg_next(" backedge from 5, latch at 5") +__msg_next(" exit from 4 to 2") +__msg_next(" exit from 5 to 6") +__msg("scev at header 2:") +__msg_next(" r6=?") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next("scev at header 3:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" scev at latch 5:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg("loop header at 2, latch not identified") +__msg("loop header at 3, header_count is [0..2] ") +__naked void nested_loop_hdr_backedge2(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 0; \ +1: r7 += 1; \ +2: r6 += 1; \ + if r7 == 2 goto 1b; \ + if r6 < 2 goto 2b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Logical nest: for (r6 = 0; r6 < 3; r6++) + * for (r7 = 0; r7 < 4; r7++) r8++; + * Both backedges target header 3, so the CFG has one loop. r6 advances only + * on the outer backedge and r7 resets there; neither has a linear SCEV. + * r8 advances on both backedges and retains its linear SCEV. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 3{{$}}") +__msg("scev at header 3:") +__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__msg_next(" r7=(any (+ r7 1) 0) / ?") +__msg_next(" r8=(+ r8 1) / (linear r8 1)") +__msg("loop header at 3, unsupported loop: multiple backedges") +__naked void nested_loops_shared_header(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 0; \ + r8 = 0; \ +1: r8 += 1; \ + r7 += 1; \ + if r7 < 4 goto 1b; \ + r7 = 0; \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + r0 = r8; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("loop header at 1, header_count is 10") +__msg("loop header at 5, header_count is 5") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void nested_loops_outer_latch_true_branch_to_inner(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; /* outer loop */ \ + r1 = 0; \ + if r0 < 10 goto 2f; /* outer latch true branch jumps into inner loop */ \ + goto 3f; \ +2: r1 += 1; /* inner loop */ \ + if r1 < 5 goto 2b; \ + goto 1b; \ +3: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Exact boundary hits for every continuation opcode. + * + * Example: pre_lt + * + * u64 r6 = 1; + * while (r6 < 7) { // r6 ∈ [1,3,5,7] + * r6 += 2; // r6 ∈ [3,5,7] + * } + */ +/* name op initial step bound expected message */ +PRE__COND_TEST (pre_lt, ">=", 1, 2, 7, "header_count is 4 ") +POST_COND_TEST (post_lt, "<", 1, 2, 7, "header_count is 3 ") +PRE__COND_TEST (pre_le, ">", 1, 2, 7, "header_count is 5 ") +POST_COND_TEST (post_le, "<=", 1, 2, 7, "header_count is 4 ") +PRE__COND_TEST (pre_gt, "<=", 9, -2, 3, "header_count is 4 ") +POST_COND_TEST (post_gt, ">", 9, -2, 3, "header_count is 3 ") +PRE__COND_TEST (pre_ge, "<", 9, -2, 3, "header_count is 5 ") +POST_COND_TEST (post_ge, ">=", 9, -2, 3, "header_count is 4 ") +PRE__COND_TEST (pre_slt, "s>=", -5, 2, 1, "header_count is 4 ") +POST_COND_TEST (post_slt, "s<", -5, 2, 1, "header_count is 3 ") +PRE__COND_TEST (pre_sle, "s>", -5, 2, 1, "header_count is 5 ") +POST_COND_TEST (post_sle, "s<=", -5, 2, 1, "header_count is 4 ") +PRE__COND_TEST (pre_sgt, "s<=", 5, -2, -1, "header_count is 4 ") +POST_COND_TEST (post_sgt, "s>", 5, -2, -1, "header_count is 3 ") +PRE__COND_TEST (pre_sge, "s<", 5, -2, -1, "header_count is 5 ") +POST_COND_TEST (post_sge, "s>=", 5, -2, -1, "header_count is 4 ") +PRE__COND_TEST (pre_ne, "==", 1, 2, 7, "header_count is 4 ") +POST_COND_TEST (post_ne, "!=", 1, 2, 7, "header_count is 3 ") +PRE__COND_TEST (pre_ne_down, "==", 9, -2, 3, "header_count is 4 ") +POST_COND_TEST (post_ne_down, "!=", 9, -2, 3, "header_count is 3 ") + +/* + * Bounds between two consecutive counter values. + * + * Example: pre_lt_round + * + * u64 r6 = 1; + * while (r6 < 8) { // r6 ∈ [1,3,5,7,9] + * r6 += 2; // r6 ∈ [3,5,7,9] + * } + */ +PRE__COND_TEST (pre_lt_round, ">=", 1, 2, 8, "header_count is 5 ") +PRE__COND_TEST (pre_le_round, ">", 1, 2, 8, "header_count is 5 ") +PRE__COND_TEST (pre_gt_round, "<=", 9, -2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_ge_round, "<", 9, -2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_slt_round, "s>=", -5, 2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_sle_round, "s>", -5, 2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_sgt_round, "s<=", 5, -2, -2, "header_count is 5 ") +PRE__COND_TEST (pre_sge_round, "s<", 5, -2, -2, "header_count is 5 ") + +/* + * Equality on the first non-strict comparison still takes a backedge. + * + * Example: pre_le_eq + * + * u64 r6 = 7; + * while (r6 <= 7) { // r6 ∈ [7,9] + * r6 += 2; // r6 ∈ [9] + * } + */ +PRE__COND_TEST (pre_le_eq, ">", 7, 2, 7, "header_count is 2 ") +PRE__COND_TEST (pre_ge_eq, "<", 3, -2, 3, "header_count is 2 ") +PRE__COND_TEST (pre_sle_eq, "s>", 1, 2, 1, "header_count is 2 ") +PRE__COND_TEST (pre_sge_eq, "s<", -1, -2, -1, "header_count is 2 ") + +/* + * Unsigned order across the sign bit. + * + * Example: pre_lt_sign + * + * u64 H = 1ULL << 63, r6 = H - 2; + * while (r6 < H + 1) { // r6 ∈ [H-2,H-1,H,H+1] + * r6 += 1; // r6 ∈ [H-1,H,H+1] + * } + */ +PRE__COND_TEST (pre_lt_sign, ">=", ITER_S64_MAX - 1, 1, ITER_S64_MIN + 1, "header_count is 4 ") +PRE__COND_TEST (pre_gt_sign, "<=", ITER_S64_MIN + 1, -1, ITER_S64_MAX - 1, "header_count is 4 ") + +/* + * The first latch comparison is false. + * + * Example: pre_lt_false + * + * u64 r6 = 7; + * while (r6 < 7) { // r6 ∈ [7] + * r6 += 2; // unreachable + * } + */ +PRE__COND_TEST (pre_lt_false, ">=", 7, 2, 7, "can't compute iterations count") +PRE__COND_TEST (pre_le_false, ">", 8, 2, 7, "can't compute iterations count") +PRE__COND_TEST (pre_gt_false, "<=", 3, -2, 3, "can't compute iterations count") +PRE__COND_TEST (pre_ge_false, "<", 2, -2, 3, "can't compute iterations count") +PRE__COND_TEST (pre_slt_false, "s>=", 1, 2, 1, "can't compute iterations count") +PRE__COND_TEST (pre_sle_false, "s>", 2, 2, 1, "can't compute iterations count") +PRE__COND_TEST (pre_sgt_false, "s<=", -1, -2, -1, "can't compute iterations count") +PRE__COND_TEST (pre_sge_false, "s<", -2, -2, -1, "can't compute iterations count") +PRE__COND_TEST (pre_ne_false, "==", 7, 2, 7, "can't compute iterations count") + +/* + * The counter moves away from the exit; bound fallback verification. + * + * Example: pre_lt_dir + * + * u64 r6 = 5, r9 = 0; + * while (++r9 <= 8 && r6 < 7) { // r6 ∈ [5,3,1,U64_MAX] + * r6 -= 2; // r6 ∈ [3,1,U64_MAX] + * } + */ +PRE__COND_TEST2(pre_lt_dir, ">=", 5, -2, 7, "can't compute iterations count") +PRE__COND_TEST2(pre_le_dir, ">", 5, -2, 7, "can't compute iterations count") +PRE__COND_TEST2(pre_gt_dir, "<=", 5, 2, 3, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_dir, "<", 5, 2, 3, "can't compute iterations count") +PRE__COND_TEST2(pre_slt_dir, "s>=", -1, -2, 1, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_dir, "s>", -1, -2, 1, "can't compute iterations count") +PRE__COND_TEST2(pre_sgt_dir, "s<=", 1, 2, -1, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_dir, "s<", 1, 2, -1, "can't compute iterations count") + +/* + * The predicted exit would wrap the counter. + * + * Example: pre_lt_wrap + * + * u64 M = U64_MAX, r6 = M - 1, r9 = 0; + * while (++r9 <= 8 && r6 < M) { // r6 ∈ [M-1,0,2,...,14] + * r6 += 2; // r6 ∈ [0,2,...,14] + * } + */ +PRE__COND_TEST2(pre_lt_wrap, ">=", ITER_U64_MAX - 1, 2, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_le_wrap, ">", ITER_U64_MAX, 1, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_gt_wrap, "<=", 1, -2, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_wrap, "<", 0, -1, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_slt_wrap, "s>=", ITER_S64_MAX - 1, 2, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_wrap, "s>", ITER_S64_MAX, 1, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sgt_wrap, "s<=", ITER_S64_MIN + 1, -2, ITER_S64_MIN, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_wrap, "s<", ITER_S64_MIN, -1, ITER_S64_MIN, "can't compute iterations count") + +/* + * The non-strict count itself would overflow u64. + * + * Example: pre_le_count_ovf + * + * u64 r6 = 0, r9 = 0; + * while (++r9 <= 8 && r6 <= U64_MAX) { // r6 ∈ [0..8] + * r6 += 1; // r6 ∈ [1..8] + * } + */ +PRE__COND_TEST2(pre_le_count_ovf, ">", 0, 1, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_count_ovf, "<", ITER_U64_MAX, -1, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_count_ovf, "s>", ITER_S64_MIN, 1, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_count_ovf, "s<", ITER_S64_MAX, -1, ITER_S64_MIN, "can't compute iterations count") + +/* + * JNE divisibility and equality across unsigned wraparound. + * + * Example: pre_ne_rem + * + * u64 r6 = 1, r9 = 0; + * while (++r9 <= 8 && r6 != 8) { // r6 ∈ [1,3,...,17] + * r6 += 2; // r6 ∈ [3,5,...,17] + * } + */ +PRE__COND_TEST2(pre_ne_rem, "==", 1, 2, 8, "can't compute iterations count") +PRE__COND_TEST2(pre_ne_rem_down, "==", 9, -2, 2, "can't compute iterations count") +PRE__COND_TEST (pre_ne_wrap_up, "==", ITER_U64_MAX - 1, 2, 0, "header_count is 2 ") +PRE__COND_TEST (pre_ne_wrap_down, "==", 1, -2, ITER_U64_MAX, "header_count is 2 ") + +/* + * Largest supported header count, sentinel, and u32 overflow. + * + * Example: pre_lt_max_hdrs + * + * u64 r6 = 0, r9 = 0; + * while (++r9 <= 8 && r6 < U32_MAX - 2) { // r6 ∈ [0..8] + * r6 += 1; // r6 ∈ [1..8] + * } + */ +PRE__COND_TEST2(pre_lt_max_hdrs, ">=", 0, 1, ITER_U32_MAX - 2, "header_count is [0..4294967294] ") +PRE__COND_TEST2(pre_lt_sentinel, ">=", 0, 1, ITER_U32_MAX - 1, "can't compute iterations count") +PRE__COND_TEST2(pre_lt_hdr_ovf, ">=", 0, 1, ITER_U32_MAX, "can't compute iterations count") + +/* + * The extreme signed step values remain valid unsigned magnitudes. + * + * Example: pre_lt_step_max + * + * u64 r6 = 0; + * while (r6 < S64_MAX) { // r6 ∈ [0,S64_MAX] + * r6 += S64_MAX; // r6 ∈ [S64_MAX] + * } + */ +PRE__COND_TEST (pre_lt_step_max, ">=", 0, ITER_S64_MAX, ITER_S64_MAX, "header_count is 2 ") +PRE__COND_TEST (pre_gt_step_min, "<=", ITER_S64_MIN, ITER_S64_MIN, 0, "header_count is 2 ") +PRE__COND_TEST (pre_ne_step_min, "==", 0, ITER_S64_MIN, ITER_S64_MIN, "header_count is 2 ") + +/* + * A zero step cannot establish a finite iteration count. + * + * Example: pre_lt_zero_step + * + * u64 r6 = 1, r9 = 0; + * while (++r9 <= 8 && r6 < 7) { // r6 ∈ [1], nine header visits + * r6 += 0; // r6 ∈ [1] + * } + */ +PRE__COND_TEST2(pre_lt_zero_step, ">=", 1, 0, 7, "can't compute iterations count") + +/* + * Signed JMP32 comparisons sign-extend the initial value, bound and step. + * Encode negative operands as positive u32 constants so the conversion matters. + * post32_sgt also checks that an exit exactly at S32_MIN is accepted. + */ +/* name op initial step bound expected message */ +POST_COND_TEST32(post32_slt, "s<", -7u, 2, -1u, "header_count is [0..3] ") +POST_COND_TEST32(post32_sle, "s<=", -7u, 2, -1u, "header_count is [0..4] ") +POST_COND_TEST32(post32_sgt, "s>", (__u32)INT32_MIN + 6, -2u, (__u32)INT32_MIN, "header_count is [0..3] ") +POST_COND_TEST32(post32_sge, "s>=", -1u, -2u, -7u, "header_count is [0..4] ") + +/* + * When computing 32-bit iteration bounds, scev.c:compute_max_iters() does + * the computation in the 64-bit domain first. For unsigned JMP32 comparisons, + * it zero-extends the initial value and bound, but sign-extends the step. + * Hence pick negative 64-bit initial values and bounds, and encode negative + * steps as positive u32 constants, so both conversions affect the computation. + * post32_lt also checks that an exit exactly at U32_MAX is accepted. + */ +POST_COND_TEST32(post32_lt, "<", -7, 2, -1, "header_count is [0..3] ") +POST_COND_TEST32(post32_le, "<=", -9, 2, -3, "header_count is [0..4] ") +POST_COND_TEST32(post32_gt, ">", -1, -2u, -7, "header_count is [0..3] ") +POST_COND_TEST32(post32_ge, ">=", -1, -2u, -7, "header_count is [0..4] ") +POST_COND_TEST32(post32_ne, "!=", -1, -2u, -7, "header_count is [0..3] ") + +/* An exit exactly at S32_MAX is accepted as well. */ +POST_COND_TEST32(post32_slt_max, "s<", INT32_MAX - 6, 2, INT32_MAX, "header_count is [0..3] ") + +/* + * The first iteration fits in 32 bits, but the predicted exit overshoots the bound + * by one and wraps. + */ +POST_COND_TEST32(post32_slt_wrap, "s<", INT32_MAX - 3, 2, INT32_MAX, "can't compute iterations count") +POST_COND_TEST32(post32_sgt_wrap, "s>", (__u32)INT32_MIN + 3, -2u, (__u32)INT32_MIN, "can't compute iterations count") +POST_COND_TEST32(post32_lt_wrap, "<", -4, 2, -1, "can't compute iterations count") + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 10 ") +__msg("loop header at 1, widening r0 to 0..9 step 1") +__msg("processed 5 insns") +__naked void post_cond_jlt(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Explore with r7=10 first, r7=4096 next. + * Should trigger invalid stack read if r7 is marked precise. + * + * r0 = 0; + * r6 = 0; + * r7 = bpf_get_prandom_u32() == 42 ? 10 : 4096; + * do { + * r0++; + * r6++; + * r3 = *(fp - r0); + * } while (r6 < r7); + * return r0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 6, header_count is 10 ") +__msg("mark_precise: frame0: parent state regs=r6,r7 stack=:") +__msg("loop header at 6, widening r0 to 0..9 step 1") +__msg("loop header at 6, header_count is 4096 ") +__msg("loop header at 6, widening r0 to 0..4095 step 1") +__msg("invalid variable-offset read from stack R2") +__naked void mark_bound_precise(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = 4096; \ + if r0 != 42 goto 1f; \ + r7 = 10; \ +1: r0 = 0; \ + r6 = 0; \ +2: r0 += 1; \ + r6 += 1; \ + r2 = r0; \ + r2 = -r2; \ + r2 += r10; \ + r3 = *(u8 *)(r2 + 0); \ + if r6 < r7 goto 2b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Explore with r6=4086 first, r6=0 next. + * Should trigger invalid stack read if r6 is marked precise. + * + * r0 = 0; + * r6 = bpf_get_prandom_u32() == 42 ? 4086 : 0; + * do { + * r0++; + * r6++; + * r3 = *(fp - r0); + * } while (r6 < 4096); + * return r0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 5, header_count is 10 ") +__msg("mark_precise: frame0: parent state regs=r6 stack=:") +__msg("loop header at 5, widening r0 to 0..9 step 1") +__msg("loop header at 5, header_count is 4096 ") +__msg("loop header at 5, widening r0 to 0..4095 step 1") +__msg("invalid variable-offset read from stack R2") +__naked void mark_base_precise(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ + if r0 != 42 goto 1f; \ + r6 = 4086; \ +1: r0 = 0; \ +2: r0 += 1; \ + r6 += 1; \ + r2 = r0; \ + r2 = -r2; \ + r2 += r10; \ + r3 = *(u8 *)(r2 + 0); \ + if r6 < 4096 goto 2b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Spill size should be taken into account when evaluating bound scev. + * + * r0 = 256; + * *(u8 *)(r10 - 8) = r0; + * r0 = 0; + * do { + * r0++; + * r1 = *(u64 *)(r10 - 8); + * if (r0 & 0x200) break; // hidden from scev, limit main pass iterations. + * } while (r0 < r1); + * return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 3:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 3, can't compute iterations count") +__naked void eval_expr_small_spill_bound(void) +{ + asm volatile (" \ + r0 = 256; \ + *(u8 *)(r10 - 8) = r0; \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x200 goto 2f; \ + if r0 < r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* Even when the saved value fits, the upper bytes of the slot are unknown. */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 3:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 3, can't compute iterations count") +__naked void eval_expr_small_spill_bound_fits(void) +{ + asm volatile (" \ + r0 = 10; \ + *(u8 *)(r10 - 8) = r0; \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x80 goto 2f; \ + if r0 < r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* + * r0 = 255; + * *(u8 *)(r10 - 8) = r0; // narrow spill + * do { + * r0 = *(u64 *)(r10 - 8); // wide load, widening should bail out here. + * r0++; + * *(u64 *)(r10 - 8) = r0; + * } while (r0 < 265); + * return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, can't compute iterations count") +__not_msg("widening") +__naked void no_widen_small_spill_counter(void) +{ + asm volatile (" \ + r0 = 255; \ + *(u8 *)(r10 - 8) = r0; \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 265 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Full-width counterpart: the counter can be widened from 5 to 14. */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, widening *fp-8 to 5..14 step 1") +__naked void widen_full_spill_counter(void) +{ + asm volatile (" \ + r0 = 5; \ + *(u64 *)(r10 - 8) = r0; \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 15 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, widening *fp-8 to 5..14 step 1") +__naked void widen_acquire_release_counter(void) +{ + asm volatile ( + "r0 = 5;" + "*(u64 *)(r10 - 8) = r0;" +"1:" + ".8byte %[load_acquire];" + "r0 += 1;" + ".8byte %[store_release];" + "if r0 < 15 goto 1b;" + "exit;" + : + : __imm_insn(load_acquire, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)), + __imm_insn(store_release, BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_0, -8)) + : __clobber_all); +} + +/* + * r6 scev is (any r6 *fp-8). The byte store leaves memory at zero, + * but the spill metadata retains 256. Widen r6 to an unknown scalar + * instead of [1,256], which would exclude the zero loaded from memory. + * + * r0 = 0; + * *(u64 *)(r10 - 8) = r0; + * r0 = 256; + * *(u8 *)(r10 - 8) = r0; // narrow spill + * r6 = 1; + * r7 = 0; + * do { + * r0 = r6; + * r6 = *(u64 *)(r10 - 8); + * r7++; + * } while (r7 < 2); + * return r0; + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("scev at header 6:") +__msg_next(" r6=*fp-8 / (any r6 *fp-8)") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("loop header at 6, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1") +__naked void widen_any_small_spill_leaf(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ + r0 = 256; \ + *(u8 *)(r10 - 8) = r0; \ + r6 = 1; \ + r7 = 0; \ +1: r0 = r6; \ + r6 = *(u64 *)(r10 - 8); \ + r7 += 1; \ + if r7 < 2 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Widening (any *fp-8 7) must allocate the slot before writing its state. + * The unallocated entry value makes the union an unknown scalar. + * + * r6 = 0; + * do { + * if (r6 == 1) + * *(u64 *)(r10 - 8) = 7; + * r0 = *(u64 *)(r10 - 8); + * r6++; + * } while (r6 < 3); + * return r0; + */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("scev at header 1:") +__msg_next("r6=(+ r6 1) / (linear r6 1)") +__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg("loop header at 1, widening r6 to 0..2 step 1") +__msg("2: (7a) *(u64 *)(r10 -8) = 7 ; R10=fp0 fp-8=7 loop_stack=1") +__naked void widen_any_unallocated_slot(void) +{ + asm volatile (" \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* A spilled zero entry contributes zero to the union. */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("loop header at 2, widening *fp-8 to 0..7 step 1") +__naked void widen_any_zero_spill(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* A byte store inside the slot leaves STACK_MISC and an unknown scalar value. */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("scev at header 2:") +__msg_next("r6=(+ r6 1) / (linear r6 1)") +__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg("loop header at 2, widening r6 to 0..2 step 1") +__msg("3: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar() R10=fp0 fp-8=m???????") +__naked void widen_any_misc_slot(void) +{ + asm volatile (" \ + *(u8 *)(r10 - 1) = 1; \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + r1 = *(u64 *)(r10 - 8); \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +__naked __noinline __used static int probe_read_stack(void) +{ + asm volatile ( + "r2 = 8;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "exit;" + : + : __imm(bpf_probe_read_kernel) + : __clobber_all); +} + +/* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */ +SEC("tracepoint") +__success +__log_level(2) +__msg("scev at header 4:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" *fp-8=?") +__msg("loop header at 4, widening r6 to 0..9 step 1") +__naked void widen_subprog_probe_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call probe_read_stack;" + "r6 = 0;" +"1:" + "r0 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call probe_read_stack;" + "r6 += 1;" + "if r6 < 10 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* An induction variable whose range overflows s64 is widened to the full range. */ +SEC("socket") +__success +__log_level(2) +__msg("loop header at 4, header_count is 10") +__msg("loop header at 4, widening r1 to 0..9 step 1") +__msg("loop header at 4, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void widen_overflowing_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 |= 1; /* not an unknown scalar, those need no widening */ \ + r1 = 0; \ +1: r6 += 24; \ + r1 += 1; \ + if r1 < 10 goto 1b; \ + r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The bound slot is unallocated at the first loop header. SCEV must + * treat it as an unknown scalar. JSET limits main-pass iterations. + * + * r0 = 0; + * do { + * r0++; + * r1 = *(u64 *)(r10 - 8); + * if (r0 & 0x80) break; + * } while (r0 < r1); + * return 0; + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 1, can't compute iterations count") +__not_msg("widening") +__naked void eval_expr_unallocated_stack_bound(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x80 goto 2f; \ + if r0 < r1 goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, header_count is 8") +__msg("loop header at 3, widening r0 to 2..9 step 1") +__msg("3: R0=scalar(smin=umin=smin32=umin32=2,smax=umax=smax32=umax32=9,{{.*}})") +__msg("processed 6 insns") +__naked void post_cond_jlt_with_base(void) +{ + asm volatile (" \ + r7 = 10 ll; /* ldimm64 for a twist */ \ + r0 = 2; \ +1: r0 += 1; \ + if r0 < r7 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 4 ") +__naked void latch_base_differs_from_step(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r1 = r0; \ + r1 += 1; \ + if r1 >= 10 goto 2f; \ + r0 += 4; \ + goto 1b; \ +2: r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 11") +__msg("loop header at 1, widening r0 to 0..10 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})") +__msg("processed 5 insns") +__naked void post_cond_jle(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 <= 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 10") +__msg("loop header at 1, widening r0 to 0..9 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=9,{{.*}})") +__msg("processed 6 insns") +__naked void post_cond_jge(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 >= 10 goto 2f; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 11") +__msg("loop header at 1, widening r0 to 0..10 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})") +__msg("processed 6 insns") +__naked void pre_cond_jge(void) +{ + asm volatile (" \ + r0 = 0; \ +1: if r0 >= 10 goto 2f; \ + r0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("1: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("2: (55) if r0 != 0x3 goto pc-2") +__msg("3: (95) exit") +__msg("loop header at 1, clamping r0") +__msg("from 2 to 1: safe") +__msg("processed 5 insns") +__naked void post_cond_jne(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 != 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 -1) / (linear r0 -1)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)") +__msg("loop header at 1, header_count is 3 ") +__msg("loop header at 1, widening r0 to 1..3 step 1") +__msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) loop_stack=1") +__msg("loop header at 1, clamping r0 to 1..2 (s32 1..2) step 1") +__msg("processed 5 insns") +__naked void post_cond_jne_neg_step(void) +{ + asm volatile (" \ + r0 = 3; \ +1: r0 += -1; \ + if r0 != 0 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" scev at latch 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=3,{{.*}})") +__msg("1: (15) if r0 == 0x3 goto pc+2 ; R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("2: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("3: (05) goto pc-3") +__msg("loop header at 1, clamping r0") +__msg("1: safe") +__msg("from 1 to 4: R0=3") +__msg("4: R0=3") +__msg("4: (95) exit") +__msg("processed 6 insns") +__naked void pre_cond_je1(void) +{ + asm volatile (" \ + r0 = 0; \ +1: if r0 == 3 goto 2f; \ + r0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(zext32 (+ r0 1)) / (zext32 (linear r0 1))") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(zext32 (+ r0 1)) / (zext32 (linear (+ r0 1) 1))") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("1: (04) w0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("2: (56) if w0 != 0x3 goto pc-2") +__msg("3: (95) exit") +__msg("loop header at 1, clamping r0") +__msg("from 2 to 1: safe") +__msg("processed 5 insns") +__naked void post_cond_jne32(void) +{ + asm volatile (" \ + w0 = 0; \ +1: w0 += 1; \ + if w0 != 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* w0 ∈ {0xffffffff, 2, 5}, w0 % 3 ∈ {0, 2, 2} ⇒ step 3 can't be preserved */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 1:") +__msg_next(" r0=(zext32 (+ r0 3)) / (zext32 (linear r0 3))") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(zext32 (+ r0 3)) / (zext32 (linear (+ r0 3) 3))") +__msg("entering loop 1") +__msg_next("loop header at 1, header_count is 2") +__msg("loop header at 1, widening r0 to 0..0xffffffff (s32 -1..2) step 1") +__msg("1: R0=scalar(smin=0,smax=umax=0xffffffff,smin32=-1,smax32=2,{{.*}})") +__naked void post_cond_jne32_overflow(void) +{ + asm volatile (" \ + w0 = 0xffffffff; \ +1: w0 += 3; \ + if w0 != 5 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * w0 ∈ {0, 3, ..., 0xffffffff, 2, 5}, w0 % 3 ∈ {0, 0, ..., 0, 2, 2} + * ⇒ step 3 can't be preserved. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 1431655767") +__msg("loop header at 2, widening r0 to 0..0xffffffff step 1") +__naked void post_cond_jne32_full_range(void) +{ + asm volatile (" \ + w0 = 0; \ + w1 = 0; \ +1: w0 += 3; \ + w1 += 1; \ + if w1 != 1431655767 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* r0 ∈ {-6, 0xfffffff7, 0xfffffff4}, r0 % 3 ∈ {0, 1, 1} ⇒ step 3 can't be preserved. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 2") +__msg("loop header at 1, widening r0 to -6..0xfffffffa (s32 -9..-6) step 1") +__naked void post_cond_jne32_base_mismatch(void) +{ + asm volatile (" \ + r0 = -6; \ +1: w0 += -3; \ + if w0 != -12 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r0 ∈ {0, 0xffffffff, 0xfffffffe}, + * ⇒ the 64-bit header bounds are [0, U32_MAX], 32-bit header bounds are [-1, 0]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 2") +__msg("loop header at 1, widening r0 to 0..0xffffffff (s32 -1..0) step 1") +__naked void post_cond_jne32_neg_step(void) +{ + asm volatile (" \ + w0 = 0; \ +1: w0 += -1; \ + if w0 != -2 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r0 ∈ {S64_MIN + 1, 3, 5}, r0 % 2 ∈ {1, 1, 1} + * ⇒ step 2 and base 1 can be preserved. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 2") +__msg("loop header at 2, widening r0 to 0x8000000000000000..0x7fffffffffffffff (s32 1..3) step 2") +__msg("2: R0=scalar({{.*}}step=1+2)") +__naked void post_cond_jne32_signed_boundary(void) +{ + asm volatile (" \ + r0 = %[initial] ll; \ +1: w0 += 2; \ + if w0 != 5 goto 1b; \ + exit; \ +" : + : __imm_const(initial, ITER_S64_MIN + 1) + : __clobber_all); +} + +/* The pre-condition also visits the header with the terminating value 3. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 4") +__msg("loop header at 1, widening r0 to 0..3 (s32 0..3) step 1") +__msg("loop header at 1, clamping r0") +__msg("4: R0=3") +__naked void pre_cond_je32(void) +{ + asm volatile (" \ + w0 = 0; \ +1: if w0 == 3 goto 2f; \ + w0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* ALU32 keeps r0 in [0, U32_MAX], so the 64-bit comparison never becomes false. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 4, can't compute iterations count") +__naked void post_cond_alu32_jmp64(void) +{ + asm volatile (" \ + w0 = 0; \ + r1 = 0x100000003 ll; \ + r9 = 0; \ +1: " + LIMIT_ITERATIONS + "w0 += 1; \ + if r0 != r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* The signed 32-bit comparison is always true, even when r0 exceeds INT32_MAX. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't compute iterations count") +__naked void post_cond_alu64_jmp32(void) +{ + asm volatile (" \ + r0 = %[initial]; \ + r9 = 0; \ +1: " + LIMIT_ITERATIONS + "r0 += 1; \ + if w0 s<= %[bound] goto 1b; \ +2: exit; \ +" : + : __imm_const(initial, INT32_MAX - 1), + __imm_const(bound, INT32_MAX) + : __clobber_all); +} + +/* ALU32 values above INT32_MAX remain positive in a signed 64-bit comparison. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, header_count is 3") +__msg("loop header at 3, widening r0 to 0xfffffffc..0xfffffffe (s32 -4..-2) step 1") +__naked void post_cond_alu32_jmp64_bounded(void) +{ + asm volatile (" \ + w0 = -4; \ + r1 = 0xffffffff ll; \ +1: w0 += 1; \ + if r0 s< r1 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* The negative 64-bit counter and its exit value fit in the signed 32-bit range. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 3") +__msg("loop header at 1, widening r0 to -6..-2 step 2") +__naked void post_cond_alu64_jmp32_bounded(void) +{ + asm volatile (" \ + r0 = -6; \ +1: r0 += 2; \ + if w0 s< 0 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is [0..3] ") +__naked void one_backedge_two_exits(void) +{ + asm volatile (" \ + r6 = 0; \ +1: call %[bpf_get_prandom_u32]; \ + if r0 == 0 goto 2f; \ + r6 += 1; \ + if r6 != 3 goto 1b; \ +2: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 11:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=(+ r1 2) / (linear r1 2)") +__msg_next(" scev at latch 16:") +__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)") +__msg_next(" r1=(+ r1 2) / (linear (+ r1 2) 2)") +__msg("loop header at 11, widening r0") +__msg("loop header at 11, widening r1") +__msg("11: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=7,var_off=(0x0; 0x7)) R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=14,var_off=(0x0; 0xe),step=0+2)") +/* loop exit */ +__msg("16: (a5) if r0 < 0x8 goto pc-6") +__msg("exiting loop 11") +__msg("17: (95) exit") +/* second iteration */ +__msg("loop header at 11, clamping r0") +__msg("loop header at 11, clamping r1") +/* iteration convergence */ +__msg("from 16 to 11: safe") +__not_msg("{{^}}11:") +/* map lookup error path */ +__msg("from 7 to 17: safe") +__naked void correlated_regs(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r0 = 0; \ + r1 = 0; \ +1: r2 = r6; \ + r2 += r1; \ + *(u8 *)(r2 + 0) = 1; \ + r0 += 1; \ + r1 += 2; \ + if r0 < 8 goto 1b; \ +2: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * k = 0 + * for (i = 0; i < 4; i++): + * for (j = 0; j < 4; j++): + * k += 1 + * k <<= 1 // make SCEV construction not possible + * k >>= 1 + * map[k] = 1 // make k precise + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("19: (72) *(u8 *)(r4 +0) = 1 ; R4=map_value(id={{.*}},map=map,ks=4,vs=1024,imm=16)") +__not_msg("19: ") +__msg("processed 106 insns") +__naked void nested_loops_precise_var1(void) +{ + asm volatile (" \ + *(u64*)(r10 - 8) = 0; \ + r1 = %[map] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r1 = 0; \ + r3 = 0; \ + /* outer loop */ \ +1: r2 = 0; \ + /* inner loop */ \ +2: r2 += 1; \ + r3 += 1; \ + r3 <<= 1; \ + r3 >>= 1; \ + if r2 < 4 goto 2b; \ + r1 += 1; \ + if r1 < 4 goto 1b; \ + r4 = r0; \ + r4 += r3; \ + *(u8 *)(r4 + 0) = 1; \ + r0 = 0; \ +3: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 2, header_count is 100 ") +__msg("loop header at 4, header_count is [0..100] ") +__msg("loop header at 6, header_count is [0..100] ") +__msg("processed 20 insns") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void nested_loop_with_two_exits(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ +1: r6 += 1; \ + r7 = 0; \ +2: r7 += 1; \ + r8 = 0; \ +3: r8 += 1; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto +1; \ + goto 4f; \ + if r8 < 100 goto 3b; \ + if r7 < 100 goto 2b; \ +4: if r6 < 100 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__naked void exit_loop_into_loop_header(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 0; \ +loop_a_%=: \ + r1 += 1; \ + if r1 < 10 goto loop_a_%=; \ +loop_b_%=: \ + r2 += 1; \ + if r2 < 10 goto loop_b_%=; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * This exercises verifier.c:loop_stack_{pop,push}() implementation, + * at 'goto d' loops 'b' and 'a' have to be popped from stack, + * while loops 'c' and 'd' have to be pushed to stack. + * + * loop a: // header 5 + * loop b: // header 6 + * if (rand) goto d; // 8 -> 12, side entry into inner loop d + * ... + * loop c: // header 11 + * loop d: // header 12 + * ... + */ +SEC("xdp") +__log_level(2) +__msg("loop at 5") +__msg("loop at 6, nested in 5") +__msg("loop at 11, irreducible") +__msg("loop at 12, nested in 11") +/* entry via if r0 == 5 goto d_%= false branch */ +__msg("loop header at 12, header_count is 3 ") +__msg("loop header at 12, widening r9 to 0..2 step 1") +__msg("12: R8=1 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) loop_stack=11,12") +/* entry via if r0 == 5 goto d_%= true branch */ +__msg("loop header at 12, header_count is 3 ") +__msg("loop header at 12, widening r9 to 0..2 step 1") +__msg("from 8 to 12: R8=0 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) R10=fp0 loop_stack=11,12") +__naked void enter_nested_loop_from_side(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ + r7 = 0; \ + r8 = 0; \ + r9 = 0; \ +a_%=: r6 += 1; \ +b_%=: r7 += 1; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 5 goto d_%=; \ + if r7 < 3 goto b_%=; \ + if r6 < 3 goto a_%=; \ +c_%=: r8 += 1; \ +d_%=: r9 += 1; \ + if r9 < 3 goto d_%=; \ + if r8 < 3 goto c_%=; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loop A (counter in r6) contains irreducible loop B (header 5), + * which contains C (counter in r9). + * Entering B at 'body' saves and restores r6, making it appear invariant. + * Entering B at 'alternate' skips the save and modifies r6 instead. + * Hence A must not infer a SCEV expression for r6. + * SCEV expression for r9 in C should still be computed. + * + * 0: r6 = 0; + * do { // A + * 1: r6++; + * 2: r0 = bpf_get_prandom_u32(); + * 3: r7 = 0; + * 4: if (r0 > 5) goto alternate; + * 5: B: r8 = r6; // B + * 6: goto body; + * 7: alternate: + * r8 = r6; + * 8: r8++; + * 9: body: + * r6 = r8; + * 10: r9 = 0; + * do { // C + * 11: r9++; + * 12: } while (r9 < 3); + * 13: r7++; + * 14: if (r7 < 4) goto B; + * 15: } while (r6 < 4); + * 16: r0 = 0; + * 17: return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 5, nested in 1, irreducible") +__msg("loop at 11, nested in 5") +__msg("scev at header 1:") +__msg_next(" r6=?") +__msg("scev at header 11:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg("loop header at 11, widening r9 to 0..2 step 1") +__not_msg("loop header at 1, widening r6") +__naked void nested_irreducible_loop(void) +{ + asm volatile (" \ + r6 = 0; \ +a_%=: r6 += 1; \ + call %[bpf_get_prandom_u32]; \ + r7 = 0; \ + if r0 > 5 goto alternate_%=; \ +b_%=: r8 = r6; \ + goto body_%=; \ +alternate_%=: \ + r8 = r6; \ + r8 += 1; \ +body_%=: \ + r6 = r8; \ + r9 = 0; \ +c_%=: r9 += 1; \ + if r9 < 3 goto c_%=; \ + r7 += 1; \ + if r7 < 4 goto b_%=; \ + if r6 < 4 goto a_%=; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loop A (counter in r6) contains irreducible loop B. + * Loop B has an exit edge from A. This is a corner case + * observed during testing: insn_aux_data[l1].need_scev == true, + * but SCEV environment is not constructed. + * + * r6 = 0; + * do { // A + * r6++; + * r0 = bpf_get_prandom_u32(); + * r7 = 0; + * if (r0 > 5) goto alternate; + * do { // B + * r7++; + * alt: r0 = bpf_get_prandom_u32(); + * l1: if (r0 == 42) + * goto exit; + * r7++; + * } while (r7 < 4); + * } while (r6 < 4); + * r0 = 0; + * exit: return r0; + */ +SEC("xdp") +__success +__log_level(2) +__naked void nested_irreducible_loop2(void) +{ + asm volatile (" \ + r6 = 0; \ +a_%=: r6 += 1; \ + call %[bpf_get_prandom_u32]; \ + r7 = 0; \ + if r0 > 5 goto alternate_%=; \ +b_%=: r7 += 1; \ +alternate_%=: \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto exit_%=; \ + r7 += 1; \ + if r7 < 4 goto b_%=; \ + if r6 < 4 goto a_%=; \ + r0 = 0; \ +exit_%=:exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Induction variable seeded from a non-constant value. r7 enters the loop as a + * range aligned to 2 (prandom & 0x6 -> {0,2,4,6}) and is incremented by a + * non-power-of-2 slope of 6. Since the entry value is not a single point, only + * the power-of-two alignment shared by the entry value and the slope can be + * guaranteed, so the widened step is 2. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 4, widening r7 to 0..18 step 2") +__msg("R7=scalar(smin=smin32=0,smax=umax=smax32=umax32=18,var_off=(0x0; 0x1e),step=0+2)") +__naked void widen_nonconst_base(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + r7 &= 0x6; \ + r6 = 0; \ +1: r7 += 6; \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * After excluding zero from {0,4,8,12}, the interval starts at 1 while the + * values remain multiples of 4. Widening over three iterations must preserve + * base 0 and include 20. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 5, header_count is 3 ") +__msg("5: R7=scalar({{.*}}smax=umax=smax32=umax32=20,{{.*}}step=0+4)") +__naked void widen_nonconst_base_refined(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + r7 &= 0xc; \ + if r7 < 1 goto 2f; \ + r8 = 0; \ +1: r7 += 4; \ + r8 += 1; \ + if r8 < 3 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Two nested loops with (1) exiting directly to (2): + * + * for (r6 = 0; r6 < 4; r6++) { + * r7 = 0; + * for (; r7 < 3; r7++) {} // (1) + * for (; r7 != 0; r7--) {} // (2) + * } + */ +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 1, widening r6 to 0..3 step 1") +__msg("loop header at 2, widening r7 to 0..2 step 1") +__msg("exiting loop 2") +__msg("entering loop 4") +__msg("loop header at 4, widening r7 to 1..3 step 1") +__naked void sibling_inner_loops(void) +{ + asm volatile (" \ + r6 = 0; \ +1: r7 = 0; \ +2: r7 += 1; \ + if r7 < 3 goto 2b; \ +3: r7 += -1; \ + if r7 != 0 goto 3b; \ + r6 += 1; \ + if r6 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("loop header at 0, can't compute iterations count") +__naked void uninit_slot_counter(void) +{ + asm volatile (" \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 10 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The assignment happens only on the second iteration: + * + * r7 = 5; + * for (r6 = 0; r6 < 3; r6++) + * if (r6 == 1) + * r7 = 10; + * if (r7 != 10) + * invalid_stack_read(); + * + * R7 is always 10 at the real exit. Widening loses the correlation between + * R6 and R7, leaving R7 in [5, 10] both at the loop header and after the loop. + * The verifier therefore rejects the possible invalid stack read. + */ +SEC("xdp") +__failure +__log_level(2) +__msg("2: {{.*}}R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))") +__msg("7: R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))") +__msg("invalid read from stack R10 off=0 size=8") +__naked void conditional_assignment_on_second_iteration(void) +{ + asm volatile (" \ + r7 = 5; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = 10; \ +2: r6 += 1; \ + goto 1b; \ +3: if r7 == 10 goto 4f; \ + r0 = *(u64 *)(r10 + 0); \ +4: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The latch bound has header SCEV (any r7 100), so it is not invariant: + * + * u64 i = 0, n = 3; + * for (;;) { + * ++i; + * if (i >= n) + * break; + * if (i == 1) + * n = 100; + * } + * + * The ANY expression must also appear at the latch, rather than a bare r7 + * that would incorrectly be treated as a loop-invariant bound. + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=(any r7 100) / (any r7 100)") +__msg_next(" scev at latch 3:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg_next(" r7=r7 / (any r7 100)") +__naked void no_widen_changing_latch_bound(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 3; \ +loop_%=: \ + r6 += 1; \ + if r6 >= r7 goto exit_%=; \ + if r6 != 1 goto next_%=; \ + r7 = 100; \ +next_%=: \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Join two stack pointer offsets via (any r7 r8), with r8 loop-invariant. + * The joined range [-16, -8] still points to initialized stack memory, + * so dereferencing r7 after the loop is safe. + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at 7, widening r7 to -16..-8 step 1") +__msg("7: {{.*}}R7=fp(smin=smin32=-16,smax=smax32=-8,") +__msg("12: R7=fp(smin=smin32=-16,smax=smax32=-8,") +__naked void conditional_stack_pointer_assignment(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = 0; \ + *(u64 *)(r10 - 8) = 0; \ + r7 = r10; \ + r7 += -16; \ + r8 = r10; \ + r8 += -8; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = r8; \ +2: r6 += 1; \ + goto 1b; \ +3: r0 = *(u64 *)(r7 + 0); \ + exit; \ +" ::: __clobber_all); +} + +/* + * Both choices are valid, but combining their offsets while retaining + * small_map's metadata would invent small_map + 32, outside its value. + * + * r6 = lookup(small_map, 0); + * r7 = lookup(map, 0) + 32; + * for (r8 = 0; r8 < 3; r8++) { + * if (r8 == 1) + * r6 = r7; + * r0 = *(u32 *)(r6 + 0); + * } + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at {{[0-9]+}}, can't widen r6, expr is") +__not_msg("widening") +__naked void no_widen_any_map_origin(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r7 = r0; \ + r7 += 32; \ + r8 = 0; \ +1: if r8 != 1 goto 2f; \ + r6 = r7; \ +2: r0 = *(u32 *)(r6 + 0); \ + r8 += 1; \ + if r8 < 3 goto 1b; \ +3: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(small_map), __imm_addr(map) + : __clobber_all); +} + +__naked __noinline __used static int any_stack_origin_callee(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r6 = r1; \ + r7 = r10; \ + r7 += -8; \ + r8 = 0; \ +1: if r8 != 1 goto 2f; \ + r6 = r7; \ +2: r8 += 1; \ + if r8 < 3 goto 1b; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +/* + * The callee chooses between caller fp-16 and its own fp-8: + * + * r6 = r1; + * r7 = fp - 8; + * for (r8 = 0; r8 < 3; r8++) + * if (r8 == 1) + * r6 = r7; + * return *(u64 *)r6; + * + * Both locations hold zero. Their offsets belong to different frames + * and cannot be combined into one stack pointer. + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at {{[0-9]+}}, can't widen r6, expr is") +__not_msg("widening") +__naked void no_widen_any_stack_origin(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = 0; \ + r1 = r10; \ + r1 += -16; \ + call any_stack_origin_callee; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3). + * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3. + */ +SEC("xdp") +__success __retval(1) +__log_level(2) +__msg("r7 += 1 {{.*}}step=1+6)") +__msg("r8 += 1 {{.*}}step=1+9)") +__msg("loop header at 9, widening r7 to 1..28 step 3") +__msg("9: {{.*}}R7=scalar({{.*}},step=1+3)") +__msg("14: R7=scalar({{.*}},step=1+3)") +__naked void conditional_scalar_assignment_gcd(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r7 = r0; \ + r7 *= 6; \ + r7 += 1; \ + r8 = r0; \ + r8 *= 9; \ + r8 += 1; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = r8; \ +2: r6 += 1; \ + goto 1b; \ +3: r0 = r7; \ + r0 %%= 3; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r7 = ctx->data; + * r8 = r7 + * for (r6 = 0; r6 < 10 && random() != 42; r7++, r6++); + * if (r7 >= ctx->data_end) + * return; + * *(r8 + 4); // At this point the loop executed unknown number of times. + * // Hence, r7 range gives no information about r8. + */ +SEC("tc") +__failure +__msg("R8 min value is outside of the allowed memory range") +__naked void break_pkt_pointers_id(void) +{ + asm volatile (" \ + r7 = *(u32*)(r1 + %[__sk_buff_data]); \ + r8 = r7; \ + r9 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r6 = 0; \ +1: call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto 3f; \ + r6 += 1; \ + r7 += 1; \ + if r6 < 10 goto 1b; \ +3: if r7 >= r9 goto 2f; \ + r0 = *(u8*)(r8 + 4); \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * A loop induction variable used to compute the base address of a store to the + * stack must not be widened: the spill offset would become varying, which the + * verifier does not track. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__naked void no_widen_stack_spill(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u64 *)(r3 + 0) = r0; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} +/* + * Same hazard across a loop nest: the outer induction variable r2 addresses a + * stack store performed inside the inner loop. The dependency is pulled up from + * the inner loop, so the outer loop must not widen r2. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__msg("loop header at 3, widening r1 to 0..1 step 1") +__naked void no_widen_stack_spill_nested(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r1 = 0; \ +2: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u64 *)(r3 + 0) = r1; \ + r1 += 1; \ + if r1 < 2 goto 2b; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The inner loop restores r6 on its backedge but exits with r6 = -1000 on break. + * Outer loop must account for the exit value instead of deriving r6 = r6_entry + n. + * On the first outer backedge r6 is -999, so the next map access is invalid. + * + * u8 *r7 = map_value; + * s64 r6 = 0; + * do { + * 1: r9 = 0; + * r0 = r7[r6]; + * while (true) { + * 2: r8 = r6; + * r6 = -1000; + * if (++r9 > 5) + * break; + * r6 = r8; + * 3: } + * r6++; + * } while (r6 < 3); + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 9:") +__msg_next(" r6=(+ ? 1) / ?") +__msg_next(" scev at latch 20:") +__msg_next(" r6=(+ ? 1) / (+ ? 1)") +__msg_next("scev at header 13:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg_next(" scev at latch 16:") +__msg_next(" r6=-1000 / -1000") +__msg_next(" r8=r6 / r6") +__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)") +__msg("R6=-999") +__msg("R1 min value is negative") +__naked void nested_loop_exit_clobbers_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -8;" + "r1 = %[map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 4f;" + "r7 = r0;" + "r6 = 0;" +"1:" + "r9 = 0;" + "r1 = r7;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" +"2:" + "r8 = r6;" + "r6 = -1000;" + "r9 += 1;" + "if r9 > 5 goto 3f;" + "r6 = r8;" + "goto 2b;" +"3:" + "r6 += 1;" + "if r6 s< 3 goto 1b;" +"4:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * The inner loop exits both nested loops with r6 unchanged, or exits only itself with r6 = -1000. + * The middle loop repairs the latter exit with a stack fill. It therefore preserves r6, + * so the outermost loop must retain r6_entry + n. + * + * r6 = 0; + * do { + * r7 = 0; + * do { + * spill = r6; + * r9 = 0; + * do { + * r8 = r6; + * if (random() & 1) + * goto next; + * r6 = -1000; + * if (++r9 >= 2) + * break; + * r6 = r8; + * } while (true); + * r6 = spill; + * } while (++r7 < 2); + * next: + * r6++; + * } while (r6 < 3); + */ +SEC("xdp") +__success __retval(3) +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("r6 = *(u64 *)(r10 -8) {{.*}}; r6 ? -> r6") +__msg("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" scev at latch 16:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg("scev at header 2:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("scev at header 4:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg(" scev at latch 9:") +__msg_next(" r8=r6 / r6") +__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)") +__naked void nested_loop_exit_preserves_reg(void) +{ + asm volatile ( + "r6 = 0;" +"1:" + "r7 = 0;" +"2:" + "*(u64 *)(r10 - 8) = r6;" + "r9 = 0;" +"3:" + "r8 = r6;" + "call %[bpf_get_prandom_u32];" + "if r0 & 1 goto 5f;" + "r6 = -1000;" + "r9 += 1;" + "if r9 >= 2 goto 4f;" + "r6 = r8;" + "goto 3b;" +"4:" + "r6 = *(u64 *)(r10 - 8);" + "r7 += 1;" + "if r7 < 2 goto 2b;" +"5:" + "r6 += 1;" + "if r6 < 3 goto 1b;" + "r0 = r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Complement of no_widen_stack_spill: a sub-register (1-byte) store to the stack + * lands as STACK_MISC and carries no tracked value, so the induction variable + * addressing it (r2) is still widened. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("widening r2 to 0..24 step 8") +__naked void widen_byte_stack_store(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u8 *)(r3 + 0) = r0; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * A fill (BPF_LDX) at a varying stack offset loses precision just like a spill, + * so the induction variable computing the load base (r2) must not be widened. + * The slots are initialized up front so the fill itself is a valid read. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 6, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__naked void no_widen_stack_fill(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 64) = r0; \ + *(u64 *)(r10 - 56) = r0; \ + *(u64 *)(r10 - 48) = r0; \ + *(u64 *)(r10 - 40) = r0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + r4 = *(u64 *)(r3 + 0); \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * struct xdp_md *slots[2] = { ctx, ctx }; + * + * r7 = 0; + * for (r8 = 0; r8 < 2; r8++) { + * value = slots[r7]->data; + * if (i == 0) + * r7 = 1; + * } + * + * SCEV expression for r7 is (any r7 8) and r7 is used to + * address the stack memory. Avoid widening it, otherwise + * verifier won't know the type of the slots[r7] expression. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 4:") +__msg_next("r7=(any r7 8) / (any r7 8)") +__msg("loop header at 4, can't widen r7, expr is {{.*}}, requires exact stack-offset tracking") +__naked void no_widen_stack_fill_any(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = r1; \ + *(u64 *)(r10 - 8) = r1; \ + r7 = 0; \ + r8 = 0; \ +1: r2 = r10; \ + r2 += -16; \ + r2 += r7; \ + r3 = *(u64 *)(r2 + 0); /* slots[r7] */ \ + r0 = *(u32 *)(r3 + 0); /* slots[r7]->data */ \ + if r8 != 0 goto 2f; \ + r7 = 8; /* conditionally update r7 */ \ +2: r8 += 1; \ + if r8 < 2 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * A dynptr/iter/irq/res_spin_lock call initializes a stack object through a + * pointer argument, which acts like a spill base: the induction variable + * computing that argument's varying stack offset must not be widened, otherwise + * the slot can't be resolved. Here each iteration constructs an xdp dynptr at + * &dptrs[i]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("can't widen {{.*}}, requires exact stack-offset tracking") +#ifndef __clang__ +/* The issue is unrelated to SCEV */ +__skip("GCC emits a stack-pointer loop condition the verifier cannot resolve") +#endif +int no_widen_dynptr_kfunc_arg(struct xdp_md *ctx) +{ + struct bpf_dynptr dptrs[4]; + int i; + +#ifdef __clang__ +#pragma clang loop unroll(disable) +#else +#pragma GCC unroll 0 +#endif + for (i = 0; i < 4; i++) + bpf_dynptr_from_xdp(ctx, 0, &dptrs[i]); + + return 0; +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to -16..-13 step 1") +__naked void ptr_stack(void) +{ + asm volatile (" \ + r6 = r10; \ + r6 += -16; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__naked __noinline __used static int ptr_stack_callee(void) +{ + asm volatile (" \ + r6 = r10; \ + r6 += -16; \ + r7 = r1; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; /* bound main pass iteration, but hide it from SCEV */ \ + if r6 < r7 goto 1b; /* r6 and r7 are from different frames, can't be widened */ \ +4: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* The end pointer belongs to the caller's frame. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (fp), r7 (fp)") +__not_msg("widening r6") +__naked void ptr_stack_other_frame(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -12; \ + call ptr_stack_callee; \ + exit; \ +" ::: __clobber_all); +} + +/* Metadata and packet data have different origins. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (pkt_meta), r7 (pkt)") +__not_msg("widening r6") +__naked void ptr_packet_meta_other_origin(void) +{ + asm volatile (" \ + r6 = *(u32 *)(r1 + %[data_meta]); \ + r7 = *(u32 *)(r1 + %[data]); \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: r0 = 0; \ + exit; \ +" : + : __imm_const(data, offsetof(struct xdp_md, data)), + __imm_const(data_meta, offsetof(struct xdp_md, data_meta)) + : __clobber_all); +} + +/* Two lookups in a 1-element array count as the same memory origin. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_map_value_two_lookups(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(small_map) + : __clobber_all); +} + +/* + * For inner array maps with BPF_F_INNER_MAP flag the actual array size + * might differ from metadata in map declaration. Hence, r6 and r7 below + * cannot be directly compared / iterations count can't be inferred. + * + * r8 = bpf_map_lookup_elem(outer_map, ...); // inner map + * r6 = bpf_map_lookup_elem(r8, ...); + * r7 = bpf_map_lookup_elem(r8, ...); + * r7 += 4; + * do { r6++; ... } while (r6 < r7); + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__not_msg("widening r6") +__naked void ptr_map_value_two_lookups_inner_map(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[outer_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r8 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = r8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r1 = r8; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 2f; /* main pass iteration guard */ \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(outer_map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_map_value(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +/* Equal offsets into different maps do not establish a common origin. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__not_msg("widening r6") +__naked void ptr_map_value_other_map(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[other_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map), __imm_addr(other_map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_mem(void) +{ + asm volatile (" \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 2f; \ + r8 = r0; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ + r1 = r8; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), __imm_addr(ringbuf) + : __clobber_all); +} + +/* Separate reservations have different pointer IDs. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (ringbuf_mem), r7 (ringbuf_mem)") +__not_msg("widening r6") +__naked void ptr_mem_other_reservation(void) +{ + asm volatile (" \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 3f; \ + r8 = r0; \ + r6 = r0; \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 2f; \ + r9 = r0; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: r1 = r9; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +2: r1 = r8; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +3: r0 = 0; \ + exit; \ +" : + : __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), __imm_addr(ringbuf) + : __clobber_all); +} + +SEC("iter/bpf_map_elem") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_buf(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + %[value]); \ + if r6 == 0 goto 2f; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(value, offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +/* Separate context loads receive different pointer IDs. */ +SEC("iter/bpf_map_elem") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (buf), r7 (buf)") +__not_msg("widening r6") +__naked void ptr_buf_other_load(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + %[value]); \ + if r6 == 0 goto 2f; \ + r7 = *(u64 *)(r1 + %[value]); \ + if r7 == 0 goto 2f; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(value, offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +/* A pointer-versus-scalar latch must not establish an iteration count. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r1 (map_value), scalar 4") +__not_msg("widening r1") +__naked void ptr_vs_scalar_no_widen(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto out_%=; \ + r1 = r0; \ + r0 = 0; \ +loop_%=: \ + r1 += 1; \ + r0 += 1; \ + if r0 & 8 goto out_%=; \ + if r1 < 4 goto loop_%=; \ +out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* A nullable pointer must not be widened before its null check. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 9, can't widen r1, expr is (+ r1 8)") +__naked void no_widen_maybe_null(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r6 = 0; \ +loop_%=: \ + if r1 == 0 goto exit_%=; \ + r1 += 8; \ + r6 += 1; \ + if r6 < 3 goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * A terminating inner loop does not prove termination of the outer loop. + * + * u64 r7 = 0; + * while (bpf_get_prandom_u32() != 42) { + * r7 = (r7 + 1) & 0xf; + * u64 r8 = 0; + * do { + * r8++; + * } while (r8 < 3); + * } + * return 0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("infinite loop detected at insn 6") +__naked void nested_terminating_prune(void) +{ + asm volatile (" \ + r7 = 0; \ +outer_%=: \ + r7 += 1; \ + r7 &= 0xf; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto out_%=; \ + r8 = 0; \ +inner_%=: \ + r8 += 1; \ + if r8 < 3 goto inner_%=; \ + goto outer_%=; \ +out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Returning directly from the inner loop to the outer header changes the + * active stack from [1, 4] to [1]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 1{{$}}") +__msg("loop at 4, nested in 1") +__msg("entering loop 1") +__msg("entering loop 4") +__msg("exiting loop 4") +__not_msg("entering loop 1") +__msg("from 4 to 1:") +__msg("loop_stack=1{{$}}") +__naked void nested_loop_backedge_to_outer_header(void) +{ + asm volatile (" \ + r6 = 0; \ +outer_%=: \ + if r6 == 3 goto exit_%=; \ + r6 += 1; \ + r7 = 0; \ +inner_%=: \ + if r7 == 2 goto outer_%=; \ + r7 += 1; \ + goto inner_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Induction variable as the src operand of the latch compare. */ +SEC("socket") +__success +__log_level(2) +__msg("loop header at 2, header_count is 10") +__msg("loop header at 2, widening r1 to 0..9 step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void post_cond_ivar_src(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 10; \ +1: r1 += 1; \ + if r2 != r1 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Early exit dominates the latch and gives a tighter bound. */ +SEC("socket") +__success +__log_level(2) +__msg(" backedge from 4, latch at 4, dominating exits at 2") +__msg("loop header at 2, header_count is [0..15]") +__msg("loop header at 2, widening r1 to 0..14 step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void dominating_exit_bounds_count(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 14; \ +1: if r2 == r1 goto 2f; \ + r1 += 1; \ + if r1 != 63 goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index 95f3239..d6cbe1e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c
@@ -9,7 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_SDIV_SMOD)) SEC("socket") __description("SDIV32, non-zero imm divisor, check 1") @@ -1271,6 +1271,7 @@ __naked void smod32_int_min_mod_neg2_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index bf9f6fb..d8e6593 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c
@@ -148,7 +148,7 @@ l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_type]); \ SEC("cgroup/skb") __description("bpf_sk_fullsock(skb->sk): no !skb->sk check") -__failure __msg("type=sock_common_or_null expected=sock_common") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void sk_no_skb_sk_check_1(void) { @@ -504,7 +504,7 @@ l1_%=: r0 = *(u32*)(r0 + %[bpf_sock_rx_queue_mapping__end]);\ SEC("cgroup/skb") __description("bpf_tcp_sock(skb->sk): no !skb->sk check") -__failure __msg("type=sock_common_or_null expected=sock_common") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void sk_no_skb_sk_check_2(void) { @@ -612,6 +612,54 @@ l1_%=: r0 = *(u64*)(r0 + %[bpf_tcp_sock_bytes_acked__end]);\ } SEC("cgroup/skb") +__description("bpf_tcp_sock(skb->sk): narrow load of tp->bytes_received") +__failure __msg("invalid tcp_sock access") +__failure_unpriv +__naked void skb_sk_tp_bytes_received_narrow(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: call %[bpf_tcp_sock]; \ + if r0 != 0 goto l1_%=; \ + exit; \ +l1_%=: r0 = *(u32*)(r0 + %[off]); \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tcp_sock), + __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(off, offsetof(struct bpf_tcp_sock, bytes_received) + 4) + : __clobber_all); +} + +SEC("cgroup/skb") +__description("bpf_tcp_sock(skb->sk): narrow load of tp->bytes_acked") +__failure __msg("invalid tcp_sock access") +__failure_unpriv +__naked void skb_sk_tp_bytes_acked_narrow(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: call %[bpf_tcp_sock]; \ + if r0 != 0 goto l1_%=; \ + exit; \ +l1_%=: r0 = *(u32*)(r0 + %[off]); \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tcp_sock), + __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(off, offsetof(struct bpf_tcp_sock, bytes_acked) + 4) + : __clobber_all); +} + +SEC("cgroup/skb") __description("bpf_tcp_sock(bpf_sk_fullsock(skb->sk)): tp->snd_cwnd") __success __success_unpriv __retval(0) __naked void skb_sk_tp_snd_cwnd_2(void) @@ -641,7 +689,7 @@ l2_%=: r0 = *(u32*)(r0 + %[bpf_tcp_sock_snd_cwnd]); \ SEC("tc") __description("bpf_sk_release(skb->sk)") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_skb_sk(void) { asm volatile (" \ @@ -658,7 +706,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("bpf_sk_release(bpf_sk_fullsock(skb->sk))") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_fullsock_skb_sk(void) { asm volatile (" \ @@ -682,7 +730,7 @@ l1_%=: r1 = r0; \ SEC("tc") __description("bpf_sk_release(bpf_tcp_sock(skb->sk))") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_tcp_sock_skb_sk(void) { asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 39a1766..4d13748 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
@@ -1277,6 +1277,8 @@ __description("stack_noperfmon: reject read of invalid slots") __success __caps_unpriv(CAP_BPF) __failure_unpriv __msg_unpriv("invalid read from stack off -8+1 size 8") +__msg_unpriv("Verification failed: Memory Safety: Uninitialized stack read") +__msg_unpriv("Initialize every byte in the stack range before reading it") __naked void stack_noperfmon_reject_invalid_read(void) { asm volatile (" \ @@ -1344,6 +1346,47 @@ __naked void old_imprecise_scalar32_vs_cur_stack_misc(void) : __clobber_all); } +SEC("socket") +__description("stack_noperfmon: reject non-fetch atomic on narrow spill") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv __msg_unpriv("invalid read from stack off -8+4 size 8") +__naked void stack_noperfmon_reject_atomic_on_narrow_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + /* A non-fetch atomic reads all 8 bytes of the slot. */ + "lock *(u64 *)(r10 - 8) += r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("stack_noperfmon: reject helper read of narrow spill") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv __msg_unpriv("invalid read from stack R2 off -8+4 size 8") +__naked void stack_noperfmon_reject_helper_read_of_narrow_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + "r1 = %[map_ringbuf] ll;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "r4 = 0;" + "call %[bpf_ringbuf_output];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ringbuf_output), + __imm_addr(map_ringbuf) + : __clobber_all); +} + SEC("raw_tp") __success __naked void var_off_write_over_scalar_spill(void)
diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 7e0ce5d..b9f799f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c
@@ -12,7 +12,8 @@ struct { __type(value, long long); } map_hash_8b SEC(".maps"); -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used @@ -116,8 +117,8 @@ SEC("tc") __description("stack_arg: pruning with different stack arg types") __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("arg JOIN insn 9 -> 10 r1: fp0-8 + _ => fp0-8|fp0+0") -__msg("arg JOIN insn 9 -> 10 sa0: fp0-8 + _ => fp0-8|fp0+0") +__msg("arg JOIN insn 8 -> 10 r1: _ + fp0-8 => fp0-8|fp0+0") +__msg("arg JOIN insn 8 -> 10 sa0: _ + fp0-8 => fp0-8|fp0+0") __msg("R{{[0-9]}} invalid mem access 'scalar'") __naked void stack_arg_pruning_type_mismatch(void) { @@ -436,6 +437,7 @@ __naked void stack_arg_sequential_calls(void) SEC("socket") __description("stack_arg is not supported by compiler or jit, use a dummy test") +__skip("stack_arg is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index c9fe485..ab19558 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c
@@ -5,7 +5,8 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used __naked @@ -115,8 +116,8 @@ __naked void stack_arg_pruning_load_after_call(void) /* * "bad_ptr": the first arg is 'long *', which is not a recognized pointer * type for static subprogs (not ctx, dynptr, or tagged). btf_prepare_func_args() - * sets arg_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The subprog - * is marked unreliable but the call still proceeds for static subprogs. + * sets arg_slot_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The + * subprog is marked unreliable but the call still proceeds for static subprogs. */ __noinline __used __naked static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) @@ -174,6 +175,7 @@ __naked void stack_arg_read_without_write_2(void) SEC("socket") __description("stack_arg order is not supported by compiler or jit, use a dummy test") +__skip("stack_arg order is not supported by compiler or jit") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 8e8cf82..7fa7bc2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c
@@ -235,6 +235,7 @@ __naked void to_stack_check_low_1(void) SEC("socket") __description("PTR_TO_STACK check low 2") +__load_if_no_large_stack() __success __failure_unpriv __msg_unpriv("R1 stack pointer arithmetic goes out of range") __retval(42) @@ -251,7 +252,26 @@ __naked void to_stack_check_low_2(void) } SEC("socket") +__description("PTR_TO_STACK check low 2, large stack") +__load_if_large_stack() +__success __failure_unpriv +__msg_unpriv("R1 stack pointer arithmetic goes out of range") +__retval(42) +__naked void to_stack_check_low_2_large(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2049; \ + r0 = 42; \ + *(u8*)(r1 + 1) = r0; \ + r0 = *(u8*)(r1 + 1); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") __description("PTR_TO_STACK check low 3") +__load_if_no_large_stack() __failure __msg("invalid write to stack R1 off=-513 size=1") __msg_unpriv("R1 stack pointer arithmetic goes out of range") __naked void to_stack_check_low_3(void) @@ -267,6 +287,23 @@ __naked void to_stack_check_low_3(void) } SEC("socket") +__description("PTR_TO_STACK check low 3, large stack") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-2049 size=1") +__msg_unpriv("R1 stack pointer arithmetic goes out of range") +__naked void to_stack_check_low_3_large(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2049; \ + r0 = 42; \ + *(u8*)(r1 + 0) = r0; \ + r0 = *(u8*)(r1 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") __description("PTR_TO_STACK check low 4") __failure __msg("math between fp pointer") __failure_unpriv @@ -483,6 +520,7 @@ l1_%=: r0 = 42; \ SEC("socket") __description("PTR_TO_STACK stack size > 512") +__load_if_no_large_stack() __failure __msg("invalid write to stack R1 off=-520 size=8") __naked void stack_check_size_gt_512(void) { @@ -495,6 +533,21 @@ __naked void stack_check_size_gt_512(void) " ::: __clobber_all); } +SEC("socket") +__description("PTR_TO_STACK stack size > 2048") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-2056 size=8") +__naked void stack_check_size_gt_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2056; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + #ifdef __BPF_FEATURE_MAY_GOTO SEC("socket") __description("PTR_TO_STACK stack size 512 with may_goto with jit") @@ -533,4 +586,40 @@ __naked void stack_check_size_512_with_may_goto(void) } #endif +/* + * Verify that old PTR_TO_STACK state is considered a super-set of + * new PTR_TO_STACK state when new variable range is a sub-range + * of the old range, e.g. old [-72, -16] vs new [-64, -16]. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +/* + * r7 is widened to [-72, -16] at the loop header (insn 3), + * the loop body sees [-64, -8] after 'r7 += 8' + */ +__msg("loop header at 3, widening r7 to -72..-16 step 8") +__msg("R7=fp(smin=smin32=-64,smax=smax32=-8") +/* + * back-edge state is clamped to the remaining iterations and pruned + * at the header, because [-64, -16] is within the widened [-72, -16] + */ +__msg("loop header at 3, clamping r7 to -64..-16 (s32 -64..-8) step 8") +__msg("from 5 to 3: safe") +__msg("processed 9 insns") +__naked void stack_ptr_subrange_in_loop(void) +{ + asm volatile (" \ + r7 = r10; \ + r7 += -72; \ + r6 = 0; \ + r7 += 8; \ + r6 += 1; \ + if r6 < 8 goto -3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_store_release.c b/tools/testing/selftests/bpf/progs/verifier_store_release.c index 72f1eb0..0abbee5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_store_release.c +++ b/tools/testing/selftests/bpf/progs/verifier_store_release.c
@@ -290,6 +290,7 @@ __naked void store_release_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release") __success int dummy_test(void) {
diff --git a/tools/testing/selftests/bpf/progs/verifier_tailcall.c b/tools/testing/selftests/bpf/progs/verifier_tailcall.c index b4acce6..51687da 100644 --- a/tools/testing/selftests/bpf/progs/verifier_tailcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_tailcall.c
@@ -28,4 +28,61 @@ __naked void invalid_map_for_tail_call(void) : __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__used __naked +static int subprog_tail_call(void) +{ + asm volatile (" \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +/* + * A tail call unwinds only the frame of the subprog doing it, so the + * frames of its callers stay on the stack. With up to 33 tail calls in + * a chain the verifier caps the stack those frames may add up to at + * 256 bytes. + */ +SEC("tc") +__description("tail call from subprog with 240 bytes of caller stack") +__success +__naked void tail_call_caller_stack_ok(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 240) = r2; \ + call subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__description("tail call from subprog with 256 bytes of caller stack") +__failure +__msg("tail_calls are not allowed when call stack of previous frames is 256 bytes. Too large") +__naked void tail_call_caller_stack_too_large(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 256) = r2; \ + call subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 3069e70..a54b5a5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c
@@ -1035,4 +1035,41 @@ __naked void stack_write_nospec_slot_index(void) " ::: __clobber_all); } +__noinline int nospec_global_func(int x) +{ + return x + 1; +} + +__used __naked static void nospec_call_subprog(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + r1 = 0; \ + if r0 < 0x100 goto l0_%=; \ + /* executed only speculatively */ \ + r1 = r10; \ +l0_%=: call nospec_global_func; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* nospec_global_func is placed in front of nospec_call_subprog */ +SEC("socket") +__description("unpriv: nospec in front of a call") +__success __success_unpriv +__caps_unpriv(CAP_BPF) +__retval(1) +__naked void nospec_backward_call(void) +{ + asm volatile (" \ + r1 = 0; \ + call nospec_global_func; \ + call nospec_call_subprog; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_unwind.c b/tools/testing/selftests/bpf/progs/verifier_unwind.c new file mode 100644 index 0000000..5a21345 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_unwind.c
@@ -0,0 +1,1240 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* + * llvm lowers unwinding to two insns: + * invoke foo unwind pad call foo; unwind_goto pad + * resume unwind_exit + * The tests spell them out as raw insns, so they don't depend on llvm version. + * The argument of unwind_goto() is the number of insns to skip. + */ +#define unwind_goto(off) ".8byte %[insn_unwind_goto_" #off "];" +#define unwind_exit ".8byte %[insn_unwind_exit];" + +#define __imm_unwind_goto(off) \ + __imm_insn(insn_unwind_goto_##off, \ + BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, off, 0)) +#define __imm_unwind_exit \ + __imm_insn(insn_unwind_exit, \ + BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0)) + +extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; +extern void bpf_throw(__u64 cookie) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} prog_array SEC(".maps"); + +struct timer_elem { + struct bpf_timer t; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_elem); +} timer_map SEC(".maps"); + +/* The verifier doesn't know what is in there */ +unsigned long zero; +unsigned long one = 1; + +/* + * The verifier follows a static subprog, so it knows which way it leaves when + * it knows the argument. To have both ways at run time the tests pass skb->len, + * which is 64, or skb->len - 64. + * + * r1 == 0: returns 7, otherwise unwinds with r0 = 5 + */ +__naked __noinline __used +static unsigned long may_unwind(void) +{ + asm volatile ( + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* owns nothing: its pad only carries on */ +__naked __noinline __used +static unsigned long passes_on(void) +{ + asm volatile ( + "call may_unwind;" + unwind_goto(2) + "r0 += 1;" + "exit;" + unwind_exit + : + : __imm_unwind_goto(2), + __imm_unwind_exit + ); +} + +__naked __noinline __used +static unsigned long always_unwinds(void) +{ + asm volatile ( + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +__naked __noinline __used +static unsigned long never_unwinds(void) +{ + asm volatile ( + "r0 = 7;" + "exit;" + ); +} + +/* Both can unwind. This one returns 7. */ +__naked __noinline __used +unsigned long global_returns(void) +{ + asm volatile ( + "r1 = %[zero] ll;" + "r1 = *(u64 *)(r1 + 0);" + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_addr(zero), + __imm_unwind_exit + ); +} + +/* This one unwinds with r0 = 5 */ +__naked __noinline __used +unsigned long global_unwinds(void) +{ + asm volatile ( + "r1 = %[one] ll;" + "r1 = *(u64 *)(r1 + 0);" + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_addr(one), + __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: the callee returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int callee_returns(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r1 -= 64;" + "call may_unwind;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: the callee unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__naked int callee_unwinds(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call may_unwind;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: return through two frames") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(8) +__naked int two_frames_return(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r1 -= 64;" + "call passes_on;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* unwind_exit returns r0 like exit does */ +SEC("tc") +__description("unwind: unwind through two frames, r0 reaches the pad") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(15) +__naked int two_frames_unwind(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call passes_on;" + unwind_goto(1) + "exit;" + "r0 += 10;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: r6-r9 and the stack are intact in the pad") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(1111) +__naked int pad_sees_callee_saved(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r6 = 1;" + "r7 = 10;" + "r8 = 100;" + "r9 = 400;" + "*(u64 *)(r10 - 8) = 600;" + "call passes_on;" + unwind_goto(2) + "r0 = 0;" + "exit;" + "r0 = r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "r1 = *(u64 *)(r10 - 8);" + "r0 += r1;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(2) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: r1-r5 are not readable in the pad") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R3 !read_ok") +__naked int pad_reads_r3(void) +{ + asm volatile ( + "r3 = 0;" + "call always_unwinds;" + unwind_goto(1) + "exit;" + "r0 = r3;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_exit of the main prog is exit") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(3) +__naked int main_prog_unwinds(void) +{ + asm volatile ( + "r0 = 3;" + unwind_exit + : + : __imm_unwind_exit + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_exit of the main prog needs r0") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R0 !read_ok") +__naked int main_prog_unwinds_no_r0(void) +{ + asm volatile ( + unwind_exit + : + : __imm_unwind_exit + : __clobber_all + ); +} + +/* + * What follows unwind_goto is dead code. unwind_exit still has to find the pad, + * and the cpu predicts that the callee returns: there is a trap for that. + */ +SEC("tc") +__description("unwind: the callee never returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__xlated("0: call pc+4") +__xlated("1: unwind_goto pc+1") +__xlated("2: goto pc-1") +__xlated("3: r0 = 42") +__xlated("4: exit") +__naked int callee_never_returns(void) +{ + asm volatile ( + "call always_unwinds;" + unwind_goto(2) + "r0 = 1;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(2) + : __clobber_all + ); +} + +/* The pad is dead code and goes away together with unwind_goto */ +SEC("tc") +__description("unwind: the callee never unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__xlated("0: call pc+1") +__xlated("1: exit") +__xlated("2: r0 = 7") +__xlated("3: exit") +__naked int callee_never_unwinds(void) +{ + asm volatile ( + "call never_unwinds;" + unwind_goto(1) + "exit;" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: the pad drops what the frame owns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__naked int pad_unlocks(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "call %[bpf_rcu_read_lock];" + "r1 = r6;" + "call may_unwind;" + unwind_goto(3) + "call %[bpf_rcu_read_unlock];" + "r0 = 0;" + "exit;" + "call %[bpf_rcu_read_unlock];" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock), + __imm_unwind_goto(3) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: the pad is checked like any other code") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("in main prog cannot be used inside bpf_rcu_read_lock-ed region") +__naked int pad_does_not_unlock(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "call %[bpf_rcu_read_lock];" + "r1 = r6;" + "call may_unwind;" + unwind_goto(3) + "call %[bpf_rcu_read_unlock];" + "r0 = 0;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock), + __imm_unwind_goto(3) + : __clobber_all + ); +} + +/* Leaves either way with the same registers */ +__naked __noinline __used +static unsigned long same_regs(void) +{ + asm volatile ( + "r0 = 0;" + "if r1 != 0 goto 1f;" + "exit;" + "1:" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* + * After the call the two states differ only in the way the callee left. + * The second one must not be pruned, or nobody looks at the pad. + */ +SEC("tc") +__description("unwind: the way the callee left is a part of the state") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R0 invalid mem access 'scalar'") +__flag(BPF_F_TEST_STATE_FREQ) +__naked int no_pruning_after_the_call(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call same_regs;" + unwind_goto(1) + "exit;" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* Returns ctx, or unwinds with a number in r0 */ +__naked __noinline __used +static unsigned long ctx_or_unwind(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + %[len]);" + "if r2 > 100 goto 1f;" + "r0 = r1;" + "exit;" + "1:" + "r0 = 0;" + unwind_exit + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_exit + ); +} + +/* + * unwind_exit changes its return address, so the cpu mispredicts the return and + * runs what follows unwind_goto with r0 of unwind_exit. + */ +SEC("socket") +__description("unwind: what follows unwind_goto runs speculatively") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(50) /* 64 without the ethernet header */ +__success_unpriv __retval_unpriv(50) +__caps_unpriv(CAP_BPF) +#ifdef SPEC_V1 +__xlated_unpriv("unwind_goto pc+{{.*}}") +__xlated_unpriv("nospec") /* inserted to prevent `R0 invalid mem access 'scalar'` */ +__xlated_unpriv("r0 = *(u32 *)(r0 +{{.*}})") +#endif +__naked int speculative_fall_through(void) +{ + asm volatile ( + "call ctx_or_unwind;" + unwind_goto(2) + "r0 = *(u32 *)(r0 + %[len]);" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(2) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: a global subprog returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int calls_global_returns(void) +{ + asm volatile ( + "call global_returns;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: a global subprog unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(15) +__naked int calls_global_unwinds(void) +{ + asm volatile ( + "call global_unwinds;" + unwind_goto(1) + "exit;" + "r0 += 10;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* The verifier doesn't look inside */ +SEC("tc") +__description("unwind: the pad of a call of a global subprog is always checked") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R0 invalid mem access 'scalar'") +__naked int global_pad_is_checked(void) +{ + asm volatile ( + "call global_returns;" + unwind_goto(1) + "exit;" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +__naked __noinline __used +unsigned long global_never_unwinds(void) +{ + asm volatile ( + "r0 = 7;" + "exit;" + ); +} + +SEC("tc") +__description("unwind: a global subprog that cannot unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int calls_global_never_unwinds(void) +{ + asm volatile ( + "call global_never_unwinds;" + "exit;" + ::: __clobber_all + ); +} + +/* + * The verifier gets to the pad from the first call only, with r6 = 1. The second + * unwind_goto stays, since the pad is not dead code. + */ +SEC("tc") +__description("unwind: a pad shared with a call of a subprog that cannot unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int shared_pad(void) +{ + asm volatile ( + "r6 = 1;" + "call global_returns;" + unwind_goto(4) + "r6 = 2;" + "call global_never_unwinds;" + unwind_goto(1) + "exit;" + "r0 = r6;" + "exit;" + : + : __imm_unwind_goto(4), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: no unwind_goto after a call of a global subprog") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("no unwind_goto after the call of global_returns()") +__naked int global_no_unwind_goto(void) +{ + asm volatile ( + "call global_returns;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("unwind: no unwind_goto after a call of a static subprog") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_exit, but no unwind_goto after the call at insn 1") +__naked int static_no_unwind_goto(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call may_unwind;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)) + : __clobber_all + ); +} + +/* The verifier sees that it doesn't get to unwind_exit */ +SEC("tc") +__description("unwind: no unwind_goto is fine when the callee doesn't unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int static_does_not_get_there(void) +{ + asm volatile ( + "r1 = 0;" + "call may_unwind;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("unwind: callx, the callee returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int callx_returns(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r1 -= 64;" + "r6 = %[may_unwind] ll;" + "callx r6;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_addr(may_unwind), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: callx, the callee unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__naked int callx_unwinds(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r6 = %[may_unwind] ll;" + "callx r6;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_addr(may_unwind), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto after a plain insn") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog") +__naked int unwind_goto_after_mov(void) +{ + asm volatile ( + "r0 = 0;" + unwind_goto(0) + "exit;" + : + : __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto is the first insn") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 0 doesn't follow a call of a subprog") +__naked int unwind_goto_first(void) +{ + asm volatile ( + unwind_goto(0) + "r0 = 0;" + "exit;" + : + : __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto after a call of a helper") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog") +__naked int unwind_goto_after_helper(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + unwind_goto(0) + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto after a call of a kfunc") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog") +__naked int unwind_goto_after_kfunc(void) +{ + asm volatile ( + "call %[bpf_rcu_read_lock];" + unwind_goto(0) + "call %[bpf_rcu_read_unlock];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock), + __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: jump to unwind_goto") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("jump to unwind_goto at insn 3") +__naked int jump_to_unwind_goto(void) +{ + asm volatile ( + "r0 = 0;" + "if r1 != 0 goto 1f;" + "call never_unwinds;" + "1:" + unwind_goto(0) + "exit;" + : + : __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: reserved fields of unwind_goto") +__failure __msg("invalid may_goto imm 1") +__naked int unwind_goto_reserved(void) +{ + asm volatile ( + "call never_unwinds;" + ".8byte %[bad];" + "exit;" + : + : __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, 0, 1)) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unknown kind of jcond") +__failure __msg("invalid may_goto imm 0") +__naked int jcond_unknown(void) +{ + asm volatile ( + "call never_unwinds;" + ".8byte %[bad];" + "exit;" + : + : __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO + 1, 0, 0)) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unknown kind of exit") +__failure __msg("BPF_EXIT uses reserved fields") +__naked int exit_unknown(void) +{ + asm volatile ( + "r0 = 0;" + ".8byte %[bad];" + : + : __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT + 1, 0, 0)) + : __clobber_all + ); +} + +/* r1 is the index: unwinds in the second iteration */ +__naked __noinline __used +static unsigned long cb_unwinds(void) +{ + asm volatile ( + "r0 = 0;" + "if r1 != 0 goto 1f;" + "exit;" + "1:" + unwind_exit + : + : __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: callback unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("callback cannot unwind") +__naked int callback_unwinds(void) +{ + asm volatile ( + "r1 = 3;" + "r2 = %[cb_unwinds];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "exit;" + : + : __imm_ptr(cb_unwinds), + __imm(bpf_loop) + : __clobber_common + ); +} + +__naked __noinline __used +static unsigned long async_cb_unwinds(void) +{ + asm volatile ( + "r0 = 0;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* The kernel calls it. It's the top frame like the main prog: unwind_exit is exit. */ +SEC("tc") +__description("unwind: async callback unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(0) +int async_callback_unwinds(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, (void *)async_cb_unwinds); + bpf_timer_start(&val->t, 0, 0); + return 0; +} + +/* Not naked: the argument has to have a name in BTF */ +__noinline int exception_cb_unwinds(__u64 cookie) +{ + if (cookie) + asm volatile ("r0 = 7;" unwind_exit :: __imm_unwind_exit : "r0"); + return 0; +} + +SEC("tc") +__description("unwind: exception callback unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__attribute__((btf_decl_tag("exception_callback:exception_cb_unwinds"))) +__success __retval(7) +int exception_callback_unwinds(struct __sk_buff *skb) +{ + bpf_throw(skb->len); + return 0; +} + +/* What it calls can unwind as long as it doesn't let it out */ +__naked __noinline __used +static unsigned long cb_catches(void) +{ + asm volatile ( + "call may_unwind;" + unwind_goto(0) + "r0 = 0;" + "exit;" + : + : __imm_unwind_goto(0) + ); +} + +SEC("tc") +__description("unwind: callback catches") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(3) +__naked int callback_catches(void) +{ + asm volatile ( + "r1 = 3;" + "r2 = %[cb_catches];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "exit;" + : + : __imm_ptr(cb_catches), + __imm(bpf_loop) + : __clobber_common + ); +} + +/* Both leave the subprog like exit does */ +__naked __noinline __used +static unsigned long does_tail_call(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "r2 = %[prog_array] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = 7;" + "if r6 == 64 goto 1f;" + unwind_exit + "1:" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm(bpf_tail_call), + __imm_addr(prog_array), + __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: tail call in a subprog that can unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int tail_call_in_subprog(void) +{ + asm volatile ( + "call does_tail_call;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* The packet is shorter than that: ld_abs leaves the subprog with r0 = 0 */ +__naked __noinline __used +static unsigned long does_ld_abs(void) +{ + asm volatile ( + "r6 = r1;" + "r0 = *(u8 *)skb[1000];" + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: ld_abs in a subprog that can unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int ld_abs_in_subprog(void) +{ + asm volatile ( + "call does_ld_abs;" + unwind_goto(2) + "r0 += 7;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(2) + : __clobber_all + ); +} + +/* + * JITs don't emit anything for unwind_goto. It's an extable entry, where + * unwind_exit looks its return address up. + * With KASAN x86-64 JIT wraps a load in pushes, a call and pops. 'r2 = 0x1234' + * is where the patterns start, so that they don't take that call for the one + * of the subprog. + */ +SEC("tc") +__description("unwind: jit") +__success __retval(42) +__load_if_JITed() +__arch_x86_64 +__jited(" movl $0x1234, %esi") +__jited("...") /* call depth accounting */ +__jited(" callq 0x{{.*}}") +__jited(" movl $0x7, %eax") +__jited("...") +__jited(" movl $0x5, %eax") +__jited(" pushq %rax") +__jited(" pushq %rsi") +__jited(" movq 0x8(%rbp), %rdi") +__jited("...") /* call depth accounting */ +__jited(" callq 0x{{.*}}") +__jited(" movq %rax, 0x8(%rbp)") +__jited(" popq %rsi") +__jited(" popq %rax") +__arch_arm64 +__jited(" mov x1, #0x1234") +__jited(" bl {{.*}}") +__jited(" mov x8, x0") +__jited(" mov x8, #0x7") +__jited("...") +__jited(" mov x8, #0x5") +__jited(" stp x8, x1, [sp, #-0x10]!") +__jited(" ldr x0, [x29, #0x8]") +__jited("...") +__jited(" mov x10, x0") +__jited(" ldp x8, x1, [sp], #0x10") +__jited("...") +__jited(" add x30, x30, x10") +__jited(" ret") +__naked int jit(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r2 = 0x1234;" + "call may_unwind;" + unwind_goto(2) + "r0 = 7;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(2) + : __clobber_all + ); +} + +/* r1 == 0: returns 7, otherwise unwinds with r0 = 5 */ +__naked __noinline __used +static unsigned long may_unwind_big_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 200) = r1;" + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* + * x86-64 keeps the pointer to the private stack in r9 and saves it around a call. + * The value that is read back in the pad is there only when r9 is restored. + */ +SEC("raw_tp") +__description("unwind: private stack, the callee unwinds") +__success __retval(47) +__load_if_JITed() +__arch_x86_64 +__jited(" movl $0x1234, %esi") /* past the load, see above */ +__jited(" pushq %r9") +__jited("...") /* call depth accounting */ +__jited(" callq 0x{{.*}}") +__jited(" popq %r9") +__jited(" jmp L{{.*}}") +__jited(" popq %r9") +__jited(" jmp L{{.*}}") +__arch_arm64 +__naked int private_stack_unwinds(void) +{ + asm volatile ( + "*(u64 *)(r10 - 200) = 42;" + "r1 = %[one] ll;" + "r1 = *(u64 *)(r1 + 0);" + "r2 = 0x1234;" + "call may_unwind_big_stack;" + unwind_goto(1) + "exit;" + "r1 = *(u64 *)(r10 - 200);" + "r0 += r1;" + "exit;" + : + : __imm_addr(one), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("raw_tp") +__description("unwind: private stack, the callee returns") +__success __retval(49) +__load_if_JITed() +__arch_x86_64 +__arch_arm64 +__naked int private_stack_returns(void) +{ + asm volatile ( + "*(u64 *)(r10 - 200) = 42;" + "r1 = %[zero] ll;" + "r1 = *(u64 *)(r1 + 0);" + "call may_unwind_big_stack;" + unwind_goto(3) + "r1 = *(u64 *)(r10 - 200);" + "r0 += r1;" + "exit;" + "r0 = 0;" + "exit;" + : + : __imm_addr(zero), + __imm_unwind_goto(3) + : __clobber_all + ); +} + +/* + * BTF FUNC records are not generated for kfuncs referenced + * from inline assembly. These records are necessary for + * libbpf to link the program. The function below is a hack + * to ensure that BTF FUNC records are generated. + */ +void __kfunc_btf_root(void) +{ + bpf_rcu_read_lock(); + bpf_rcu_read_unlock(); +} + +char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c index 4d8273c..3166333 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c
@@ -22,8 +22,8 @@ struct { SEC("socket") __description("map element value illegal alu op, 1") -__failure __msg("R0 bitwise operator &= on pointer") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator &= on pointer") __naked void value_illegal_alu_op_1(void) { asm volatile (" \ @@ -70,8 +70,8 @@ l0_%=: exit; \ SEC("socket") __description("map element value illegal alu op, 3") -__failure __msg("R0 pointer arithmetic with /= operator") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 pointer arithmetic with /= operator") __naked void value_illegal_alu_op_3(void) { asm volatile (" \ @@ -165,6 +165,171 @@ __naked void map_ptr_illegal_alu_op(void) : __clobber_all); } +SEC("socket") +__description("tag in the low bit of a pointer, and, shift") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__naked void ptr_tag_and_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r2 = r0; \ + r2 >>= 1; \ + r3 = r0; \ + r3 |= 1; \ + r3 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("tag in the low bit of a pointer, CAP_BPF without CAP_PERFMON") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__caps_unpriv(CAP_BPF) +__naked void ptr_tag_cap_bpf(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("number op= pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 pointer arithmetic with *= operator") +__naked void number_mul_ptr(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = 7; \ + r1 *= r0; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("pointer with the tag cleared is a number") +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator |= on pointer") +__naked void ptr_tag_cleared_deref(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r0 |= 1; \ + r0 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("shift of a pointer that may be NULL") +__failure __msg("R0 pointer arithmetic on map_value_or_null prohibited, null-check it first") +__failure_unpriv +__naked void ptr_or_null_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 >>= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("and of a pointer to map") +__failure __msg("R0 pointer arithmetic on map_ptr prohibited") +__failure_unpriv +__naked void map_ptr_and(void) +{ + asm volatile (" \ + r0 = %[map_hash_48b] ll; \ + r0 &= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("32-bit and of a pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R0 32-bit pointer arithmetic prohibited") +__naked void ptr_and32(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + w0 &= 1; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + SEC("flow_dissector") __description("flow_keys illegal alu op with variable offset") __failure __msg("R7 pointer arithmetic on flow_keys prohibited")
diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index a63e336..3998849 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c
@@ -406,6 +406,7 @@ __naked void zero_sized_access_max_out_of_bound(void) SEC("lwt_in") __description("indirect variable-offset stack access, min out of bound") +__load_if_no_large_stack() __failure __msg("invalid variable-offset read from stack R2") __naked void access_min_out_of_bound(void) { @@ -433,6 +434,37 @@ __naked void access_min_out_of_bound(void) : __clobber_all); } +SEC("lwt_in") +__description("indirect variable-offset stack access, min out of bound, large stack") +__load_if_large_stack() +__failure __msg("invalid variable-offset read from stack R2") +__naked void access_min_out_of_bound_large(void) +{ + asm volatile (" \ + /* Fill the top 8 bytes of the stack */ \ + r2 = 0; \ + *(u64*)(r10 - 8) = r2; \ + /* Get an unknown value */ \ + r2 = *(u32*)(r1 + 0); \ + /* Make it small and 4-byte aligned */ \ + r2 &= 4; \ + r2 -= 2052; \ + /* \ + * add it to fp. We now have either fp-2052 or fp-2048, but\ + * we don't know which \ + */ \ + r2 += r10; \ + /* dereference it indirectly */ \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + SEC("cgroup/skb") __description("indirect variable-offset stack access, min_off < min_initialized") __success
diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 8f0c454..b5f456d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c
@@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("R1 is fp expected STRUCT task_struct") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; @@ -80,7 +80,7 @@ int BPF_PROG(get_task_exe_file_kfunc_unreleased) } SEC("lsm.s/file_open") -__failure __msg("release kfunc bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(put_file_kfunc_unacquired, struct file *file) { /* Can't release an unacquired pointer. */ @@ -128,7 +128,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_current) } SEC("lsm.s/file_open") -__failure __msg("kernel function bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") +__failure __msg("bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") int BPF_PROG(path_d_path_kfunc_type_mismatch, struct file *file) { bpf_path_d_path((struct path *)&file->f_task_work, buf, sizeof(buf));
diff --git a/tools/testing/selftests/bpf/progs/verifier_xadd.c b/tools/testing/selftests/bpf/progs/verifier_xadd.c index 05a0a55..7bde3da 100644 --- a/tools/testing/selftests/bpf/progs/verifier_xadd.c +++ b/tools/testing/selftests/bpf/progs/verifier_xadd.c
@@ -121,4 +121,31 @@ l0_%=: r0 = 42; \ " ::: __clobber_all); } +SEC("tc") +__description("xadd with variable stack offset") +__failure +__msg("variable offset stack pointer cannot be passed into helper function") +__msg("Verification failed: Memory Safety: Variable-offset stack access") +__msg("The instruction would access the stack") +__msg("Use a fixed stack offset for the instruction") +__naked void xadd_variable_stack_offset(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 16) = r1; \ + *(u64 *)(r10 - 8) = r1; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r1 = r10; \ + r1 += -16; \ + r1 += r0; \ + r2 = 1; \ + lock *(u64 *)(r1 + 0) += r2; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c index d4d0f16..ec4e0f3 100644 --- a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c +++ b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c
@@ -42,7 +42,7 @@ int wakeup_source_access_lock_fields(void *ctx) } SEC("syscall") -__failure __msg("release kfunc bpf_wakeup_sources_read_unlock expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") int wakeup_source_unlock_no_lock(void *ctx) { struct bpf_ws_lock *lock = (void *)0x1;
diff --git a/tools/testing/selftests/bpf/progs/wq_failures.c b/tools/testing/selftests/bpf/progs/wq_failures.c index 32dc882..bd30217 100644 --- a/tools/testing/selftests/bpf/progs/wq_failures.c +++ b/tools/testing/selftests/bpf/progs/wq_failures.c
@@ -48,7 +48,7 @@ __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __failure __msg(": (85) call bpf_wq_init#") /* anchor message */ -__msg("pointer in R2 isn't map pointer") +__msg("R2 type=fp expected=map_ptr") long test_wq_init_nomap(void *ctx) { struct bpf_wq *wq; @@ -98,7 +98,7 @@ __failure * is a correct bpf_wq pointer. */ __msg(": (85) call bpf_wq_set_callback#") /* anchor message */ -__msg("R1 doesn't point to a map value") +__msg("R1 type=fp expected=map_value") long test_wrong_wq_pointer(void *ctx) { int key = 0;
diff --git a/tools/testing/selftests/bpf/test_btf.h b/tools/testing/selftests/bpf/test_btf.h index e65889a..e7bc781 100644 --- a/tools/testing/selftests/bpf/test_btf.h +++ b/tools/testing/selftests/bpf/test_btf.h
@@ -7,7 +7,7 @@ #define BTF_END_RAW 0xdeadbeef #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) \ (name), (info), (size_or_type)
diff --git a/tools/testing/selftests/bpf/test_kmod.sh b/tools/testing/selftests/bpf/test_kmod.sh deleted file mode 100755 index 50dca53..0000000 --- a/tools/testing/selftests/bpf/test_kmod.sh +++ /dev/null
@@ -1,73 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 - -# Usage: -# ./test_kmod.sh [module_param]... -# Ex.: ./test_kmod.sh test_range=1,3 -# All the parameters are passed to the kernel module. - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 - -msg="skip all tests:" -if [ "$(id -u)" != "0" ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -if [ "$building_out_of_srctree" ]; then - # We are in linux-build/kselftest/bpf - OUTPUT=../../ -else - # We are in linux/tools/testing/selftests/bpf - OUTPUT=../../../../ -fi - -test_run() -{ - sysctl -w net.core.bpf_jit_enable=$1 2>&1 > /dev/null - sysctl -w net.core.bpf_jit_harden=$2 2>&1 > /dev/null - - echo "[ JIT enabled:$1 hardened:$2 ]" - shift 2 - dmesg -C - if [ -f ${OUTPUT}/lib/test_bpf.ko ]; then - insmod ${OUTPUT}/lib/test_bpf.ko "$@" 2> /dev/null - if [ $? -ne 0 ]; then - rc=1 - fi - else - # Use modprobe dry run to check for missing test_bpf module - if ! /sbin/modprobe -q -n test_bpf "$@"; then - echo "test_bpf: [SKIP]" - elif /sbin/modprobe -q test_bpf "$@"; then - echo "test_bpf: ok" - else - echo "test_bpf: [FAIL]" - rc=1 - fi - fi - rmmod test_bpf 2> /dev/null - dmesg | grep FAIL -} - -test_save() -{ - JE=`sysctl -n net.core.bpf_jit_enable` - JH=`sysctl -n net.core.bpf_jit_harden` -} - -test_restore() -{ - sysctl -w net.core.bpf_jit_enable=$JE 2>&1 > /dev/null - sysctl -w net.core.bpf_jit_harden=$JH 2>&1 > /dev/null -} - -rc=0 -test_save -test_run 0 0 "$@" -test_run 1 0 "$@" -test_run 1 1 "$@" -test_run 1 2 "$@" -test_restore -exit $rc
diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 9366a3c..6380f52 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
@@ -17,6 +17,7 @@ #include <linux/in.h> #include <linux/in6.h> #include <linux/un.h> +#include <linux/unaligned.h> #include <linux/filter.h> #include <linux/rcupdate_trace.h> #include <net/sock.h> @@ -75,6 +76,16 @@ union bpf_testmod_union_arg_2 { struct bpf_testmod_struct_arg_2 arg; }; +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + __bpf_hook_start(); noinline int @@ -186,6 +197,43 @@ bpf_testmod_test_int128_arg(__int128 a, int b, long c) bpf_testmod_test_struct_arg_result = (long)a + b + c; return bpf_testmod_test_struct_arg_result; } + +struct bpf_testmod_struct_int128 { + __int128 a; +}; + +/* + * x86-64 puts g in the first stack slot and h, aligned to 16 bytes, in + * the third and fourth, leaving a hole. arm64 leaves x7 empty and puts h + * in the first two stack slots. Either way h and i are not where the BPF + * convention packs them, right after g. + */ +noinline long +bpf_testmod_test_int128_stack(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, struct bpf_testmod_struct_int128 h, u64 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f * 6 + g * 7 + (u64)h.a * 8 + + (u64)((unsigned __int128)h.a >> 64) * 9 + i * 10; + return bpf_testmod_test_struct_arg_result; +} + +/* + * x86-64 has no register left for f after e, so f goes to the stack and g + * takes R9 back; h goes to the third stack slot and i, aligned to 16 + * bytes, to the fifth and sixth. arm64 has registers up to g, puts h in + * the first stack slot and i in the third and fourth. + */ +noinline long +bpf_testmod_test_int128_backfill(u64 a, u64 b, u64 c, u64 d, u64 e, + struct bpf_testmod_struct_arg_2 f, u64 g, u64 h, + struct bpf_testmod_struct_int128 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f.a * 6 + f.b * 7 + g * 8 + h * 9 + (u64)i.a * 10 + + (u64)((unsigned __int128)i.a >> 64) * 11; + return bpf_testmod_test_struct_arg_result; +} #endif __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) @@ -302,6 +350,15 @@ __bpf_kfunc void bpf_kfunc_trusted_task_test(struct task_struct *ptr) { } +__bpf_kfunc struct task_struct *bpf_kfunc_trusted_null_test(void) +{ + return NULL; +} + +__bpf_kfunc void bpf_kfunc_trusted_nullable_test(struct task_struct *ptr__nullable) +{ +} + __bpf_kfunc void bpf_kfunc_trusted_num_test(int *ptr) { } @@ -336,6 +393,47 @@ __bpf_kfunc void bpf_kfunc_put_default_trusted_ptr_test(struct prog_test_member */ } +#ifdef CONFIG_BPF_JIT_KASAN + +extern void kasan_poison(const void *addr, size_t size, u8 value, bool init); + +#define KASAN_SLAB_FREE 0xFB + +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, KASAN_SLAB_FREE, false); +} + +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, 0x00, false); +} +#else +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) { } +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) { } +#endif + +__bpf_kfunc struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) +{ + struct bpf_testmod_oob *p; + + /* + * Only allocate size of data (and so, voluntarily use kmalloc + * instead of kmalloc_obj), not the rest of the structure, so + * that programs under test trying to access the rest of the + * structure trigger OoB accesses + */ + p = kmalloc(sizeof(p->data), GFP_ATOMIC); + if (!p) + return NULL; + return p; +} + +__bpf_kfunc void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) +{ + kfree(oob); +} + __bpf_kfunc struct bpf_testmod_ctx * bpf_testmod_ctx_create(int *err) { @@ -408,6 +506,16 @@ static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair return 0; } +static int bpf_testmod_ops3__test_arena_member(struct bpf_testmod_arena_member_args *args) +{ + return 0; +} + +static int bpf_testmod_ops3__test_nullable(struct bpf_testmod_nullable_args *args) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, @@ -415,6 +523,8 @@ static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, .test_arena_stack = bpf_testmod_ops3__test_arena_stack, .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, + .test_arena_member = bpf_testmod_ops3__test_arena_member, + .test_nullable = bpf_testmod_ops3__test_nullable, }; static void bpf_testmod_test_struct_ops3(void) @@ -455,6 +565,58 @@ __bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) return st_ops3->test_arena_multislot(p, ptr__arena); } +/* The callback gets the kernel address of @ptr__arena behind each __arena member. */ +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_member(u64 *ptr__arena) +{ + struct bpf_testmod_arena_member_args args = { + .ptr = ptr__arena, + .nested.ptr = ptr__arena, + .u.ptr = ptr__arena, + .cookie = 0xc00c1e, + /* in the first page, which is never mapped */ + .faulting = (void *)16, + }; + + if (!st_ops3 || !st_ops3->test_arena_member) + return -EOPNOTSUPP; + return st_ops3->test_arena_member(&args); +} + +/* + * The members of struct bpf_testmod_nullable_args are trusted the way the + * verifier's BTF_TYPE_SAFE_TRUSTED() lists vmlinux members: through a struct + * of the same name with the __safe_trusted suffix in the BTF of the module. + */ +struct bpf_testmod_nullable_args__safe_trusted { + struct task_struct *task; + struct task_struct *task_or_null; +}; + +static DEFINE_PER_CPU(struct bpf_testmod_pcpu, bpf_testmod_nullable_pcpu); + +/* + * Call test_nullable() with the __nullable members unset, then set. The + * callback returns 1 when it found a task in @task_or_null, so the expected + * result is 2. + */ +__bpf_kfunc int bpf_testmod_ops3_call_test_nullable(void) +{ + struct bpf_testmod_nullable_args args = { .task = current }; + int seven = 7, ret; + + BTF_TYPE_EMIT(struct bpf_testmod_nullable_args__safe_trusted); + + if (!st_ops3 || !st_ops3->test_nullable) + return -EOPNOTSUPP; + args.self = &args; + ret = st_ops3->test_nullable(&args); + args.task_or_null = current; + this_cpu_write(bpf_testmod_nullable_pcpu.val, 0x5ca1ab1e); + args.pcpu_or_null = &bpf_testmod_nullable_pcpu; + args.int_or_null = &seven; + return ret | st_ops3->test_nullable(&args) << 1; +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -631,6 +793,12 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, #ifdef __SIZEOF_INT128__ (void)bpf_testmod_test_int128_ret(i); (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); + (void)bpf_testmod_test_int128_stack(1, 2, 3, 4, 5, 6, 7, + (struct bpf_testmod_struct_int128){ .a = ((__int128)9 << 64) | 8 }, + 10); + (void)bpf_testmod_test_int128_backfill(1, 2, 3, 4, 5, + (struct bpf_testmod_struct_arg_2){ .a = 6, .b = 7 }, 8, 9, + (struct bpf_testmod_struct_int128){ .a = ((__int128)11 << 64) | 10 }); #endif (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); @@ -683,6 +851,13 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, EXPORT_SYMBOL(bpf_testmod_test_read); ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO); +static __always_inline size_t testmod_inline_len(size_t len) +{ + /* Keep an instruction in the inline site without changing len. */ + asm volatile("" : "+m"(len)); + return len; +} + noinline ssize_t bpf_testmod_test_write(struct file *file, struct kobject *kobj, const struct bin_attribute *bin_attr, @@ -691,7 +866,7 @@ bpf_testmod_test_write(struct file *file, struct kobject *kobj, struct bpf_testmod_test_write_ctx ctx = { .buf = buf, .off = off, - .len = len, + .len = testmod_inline_len(len), }; trace_bpf_testmod_test_write_bare_tp(current, &ctx); @@ -750,7 +925,7 @@ static struct testmod_uprobe uprobe = { .consumer.ret_handler = uprobe_ret_handler, }; -static int testmod_register_uprobe(loff_t offset) +static __always_inline int testmod_register_uprobe(loff_t offset) { int err = -EBUSY; @@ -855,6 +1030,8 @@ BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_zero_offset_test, KF_ACQUIRE) BTF_ID_FLAGS(func, bpf_kfunc_nested_release_test, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_trusted_vma_test) BTF_ID_FLAGS(func, bpf_kfunc_trusted_task_test) +BTF_ID_FLAGS(func, bpf_kfunc_trusted_null_test, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_trusted_nullable_test) BTF_ID_FLAGS(func, bpf_kfunc_trusted_num_test) BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU) BTF_ID_FLAGS(func, bpf_kfunc_ret_rcu_test, KF_RET_NULL | KF_RCU_PROTECTED) @@ -867,8 +1044,14 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_member) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_nullable) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); +BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) +BTF_ID_FLAGS(func, bpf_kfunc_kasan_unpoison) +BTF_ID_FLAGS(func, bpf_testmod_oob_alloc, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_testmod_oob_free, KF_RELEASE) BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) BTF_ID_LIST(bpf_testmod_dtor_ids) @@ -954,6 +1137,188 @@ __bpf_kfunc int bpf_kfunc_call_test5(u8 a, u16 b, u32 c) return 0; } +#if defined(__x86_64__) || defined(__aarch64__) +__bpf_kfunc __int128 bpf_kfunc_call_test_i128(u64 a, u64 b) +{ + return (__int128)(((unsigned __int128)(a + b) << 64) | (a - b)); +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg(u64 a, struct prog_test_pair_arg s, u64 b) +{ + return a + s.lo * 2 + s.hi * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v) +{ + return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd(u64 a, __int128 v, u64 b) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd_many(u64 a, __int128 v, u64 b, u64 c, + u64 d, u64 e, u64 f, __int128 w) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + + b * 4 + c * 5 + d * 6 + e * 7 + f * 8 + + (u64)w * 9 + (u64)((unsigned __int128)w >> 64) * 10; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_stack_hole(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, + u64 f, u64 g, + struct prog_test_i128_arg w) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + + f * 8 + g * 9 + (u64)w.v * 10 + + (u64)((unsigned __int128)w.v >> 64) * 11; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, __int128 v) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (u64)v * 8 + (u64)((unsigned __int128)v >> 64) * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_nofit(u64 a, u64 b, u64 c, u64 d, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + s.lo * 5 + s.hi * 6; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_tail(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, u64 f) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + f * 8; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_split8(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + s.lo * 8 + s.hi * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) +{ + return s.x; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arena_arg(u64 a, u64 b, u64 c, + struct prog_test_pair_arg s, + u64 *f__arena) +{ + return a + b + c + s.lo + s.hi + *f__arena; +} + +__bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) +{ + struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(u64 addr) +{ + struct prog_test_ret_arena r = { .a = (void *)addr, .b = (void *)(addr + 4) }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(u64 addr) +{ + struct prog_test_ret_arena_mixed r = { .p = (void *)addr, .tag = 0xbeef }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(u64 addr) +{ + struct prog_test_ret_arena_untagged r = { .a = (void *)addr, .b = NULL }; + + return r; +} + +__bpf_kfunc union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(u64 addr) +{ + union prog_test_ret_arena_union r = { .a = (void *)addr }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) +{ + struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) +{ + struct prog_test_ret_ptr_arr r = { .p = { NULL, NULL } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) +{ + struct prog_test_ret_arr_struct r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) +{ + struct prog_test_ret_arr2d r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(u64 v) +{ + struct prog_test_ret_deep r = { .l1 = { .l2 = { .l3 = { .l4 = { .v = v } } } } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) +{ + struct prog_test_ret_ii r = { .a = a, .b = b }; + + return r; +} +#endif /* __x86_64__ || __aarch64__ */ + +/* + * Takes no argument on purpose: with no arguments there is nothing for the sret + * pointer to displace, so this needs no architecture guard even though it + * returns 24 bytes. + */ +__bpf_kfunc struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) +{ + struct prog_test_ret_big r = { .a = 1, .b = 2, .c = 3 }; + + return r; +} + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1021,7 +1386,7 @@ __bpf_kfunc u64 bpf_kfunc_call_stack_arg_timer(u64 a, u64 b, u64 c, u64 d, u64 e __bpf_kfunc u64 bpf_kfunc_call_stack_arg_big(u64 a, u64 b, u64 c, u64 d, u64 e, struct prog_test_big_arg s) { - return a + b + c + d + e + s.a + s.b; + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.a * 6 + s.b * 7; } static struct prog_test_ref_kfunc prog_test_struct = { @@ -1102,6 +1467,50 @@ __bpf_kfunc void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) { } +__bpf_kfunc void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) +{ + out__uninit->x0 = 1; + out__uninit->x1 = 2; + out__uninit->x2 = 3; + out__uninit->x3 = 4; +} + +__bpf_kfunc void bpf_kfunc_test_uninit_mem(void *out__uninit, u32 out__sz) +{ + memset(out__uninit, 0x2a, out__sz); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_partial(void *out__uninit, u32 out__sz) +{ + if (out__sz) + *(u8 *)out__uninit = 42; +} + +__bpf_kfunc int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) +{ + int value = get_unaligned(in); + + put_unaligned(42, out__uninit); + return value; +} + +__bpf_kfunc void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, u32 b__sz) +{ + put_unaligned(42, a__uninit); + memset(b__uninit, 0x2a, b__sz); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) +{ + put_unaligned((int)(p.lo + p.hi), out__uninit); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_stack(u64 a, u64 b, u64 c, u64 d, u64 e, + int *out__uninit) +{ + put_unaligned((int)(a + b + c + d + e), out__uninit); +} + __bpf_kfunc void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) { } @@ -1487,6 +1896,34 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) BTF_ID_FLAGS(func, bpf_kfunc_call_test4) BTF_ID_FLAGS(func, bpf_kfunc_call_test5) +#if defined(__x86_64__) || defined(__aarch64__) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_mixed) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_untagged) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_union) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd_many) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_stack_hole) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_split8) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ptr_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arena_arg) +#endif +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) @@ -1508,6 +1945,13 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_struct) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_mem) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_partial) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_alias) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_multi) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_pair) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_stack) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) @@ -1578,6 +2022,7 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { }; static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = { + .get_func_proto = bpf_base_func_proto, .is_valid_access = bpf_testmod_ops_is_valid_access, }; @@ -1592,6 +2037,8 @@ static int bpf_dummy_reg(void *kdata, struct bpf_link *link) */ if (ops->test_2) ops->test_2(4, ops->data); + if (ops->test_trampoline_stack_args) + ops->test_trampoline_stack_args(1, 2, 3, 4, 5, 6, 7, 8, 9999); return 0; } @@ -1639,6 +2086,13 @@ bpf_testmod_ops__test_return_ref_kptr(int dummy, struct task_struct *task__ref, return NULL; } +static int bpf_testmod_ops__test_trampoline_stack_args(int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9) +{ + return arg9; +} + static struct bpf_testmod_ops __bpf_testmod_ops = { .test_1 = bpf_testmod_test_1, .test_2 = bpf_testmod_test_2, @@ -1646,6 +2100,7 @@ static struct bpf_testmod_ops __bpf_testmod_ops = { .test_refcounted = bpf_testmod_ops__test_refcounted, .test_refcounted_multi = bpf_testmod_ops__test_refcounted_multi, .test_return_ref_kptr = bpf_testmod_ops__test_return_ref_kptr, + .test_trampoline_stack_args = bpf_testmod_ops__test_trampoline_stack_args, }; struct bpf_struct_ops bpf_bpf_testmod_ops = {
diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 210b919..4ba9def 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h
@@ -46,6 +46,11 @@ struct bpf_testmod_ops { struct task_struct *(*test_return_ref_kptr)(int dummy, struct task_struct *task, struct cgroup *cgrp); + /* Used to test indirect struct_ops trampoline with stack-passed arguments. */ + int (*test_trampoline_stack_args)(int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9); + /* The following fields are used to test shadow copies. */ char onebyte; struct { @@ -109,6 +114,76 @@ struct bpf_testmod_arena_pair { u64 b; }; +/* + * The qualifiers are BTF type tags for the verifier. The module gets __arena + * and __nullable from bpf_compiler_types.h. A BPF program gets __arena from + * bpf_arena_common.h, and the same definition here when it includes this + * header first, and needs __nullable and __percpu empty. User space needs all + * of them empty. + */ +#ifdef __KERNEL__ +#include <linux/bpf_compiler_types.h> +#endif +#if defined(__BPF__) && !defined(__arena) +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM) +#define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena"))) +#else +#define __arena __attribute__((btf_type_tag("arena"))) +#endif +#endif +#ifndef __arena +#define __arena +#endif +#ifndef __nullable +#define __nullable +#endif +#ifndef __percpu +#define __percpu +#endif + +struct bpf_testmod_pcpu { + int val; +}; + +struct bpf_testmod_arena_nested { + u64 __arena *ptr; +}; + +union bpf_testmod_arena_union { + u64 __arena *ptr; + u64 raw; +}; + +/* Arguments of test_arena_member(), the pointers hold kernel addresses of arena memory */ +struct bpf_testmod_arena_member_args { + /* the value the caller passed, never NULL */ + u64 __arena *ptr; + /* always NULL */ + u64 __arena __nullable *null_ptr; + /* the same value again, behind a nested struct */ + struct bpf_testmod_arena_nested nested; + /* the same value again, but a load from a union is rejected */ + union bpf_testmod_arena_union u; + /* a scalar, loaded by the same insn as @ptr in a verifier test */ + u64 cookie; + /* a bad pointer to a struct like this one, a load through it faults */ + struct bpf_testmod_arena_member_args *faulting; +}; + +/* Arguments of test_nullable(), trusted through the __safe_trusted alias in the module */ +struct bpf_testmod_nullable_args { + /* never NULL */ + struct task_struct *task; + /* NULL on the first call, set on the second */ + struct task_struct __nullable *task_or_null; + /* NULL on the first call, a per-CPU variable on the second */ + struct bpf_testmod_pcpu __percpu __nullable *pcpu_or_null; + /* NULL on the first call, set on the second */ + int __nullable *int_or_null; + /* this struct again, through a plain pointer, which the verifier probes */ + struct bpf_testmod_nullable_args *self; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); @@ -120,6 +195,10 @@ struct bpf_testmod_ops3 { u64 g, u64 h, u64 *ptr); /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); + /* Used to test __arena pointer members. */ + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); + /* Used to test __nullable pointer members. */ + int (*test_nullable)(struct bpf_testmod_nullable_args *args); }; struct st_ops_args {
diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 7d81070..3b810f1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
@@ -26,6 +26,12 @@ struct prog_test_ref_kfunc { }; #endif +#if __has_attribute(btf_type_tag) +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#else +#define __arena_tag +#endif + struct bpf_iter_testmod_seq; struct prog_test_pass1 { @@ -55,6 +61,100 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_pair_arg { /* 16 bytes: two argument registers */ + __u64 lo; + __u64 hi; +}; + +#ifdef __SIZEOF_INT128__ +struct prog_test_i128_arg { /* 16 bytes, aligned to 16 like its member */ + __int128 v; +}; +#endif + +struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */ + void *p; + __u64 x; +}; + +struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ + __u64 lo; + __u64 hi; +}; + +struct prog_test_ret_ii { /* 8 bytes: R0 only */ + int a; + int b; +}; + +struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ + void *p; + __u64 tag; +}; + +struct prog_test_ret_arena { /* 16 bytes: two arena pointers */ + void __arena_tag *a; + void __arena_tag *b; +}; + +struct prog_test_ret_arena_mixed { /* 16 bytes: an arena pointer and a scalar */ + void __arena_tag *p; + __u64 tag; +}; + +struct prog_test_ret_arena_untagged { /* 16 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +union prog_test_ret_arena_union { /* 8 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ + struct { + void *p; + } in; + __u64 tag; +}; + +struct prog_test_ret_ptr_arr { /* 16 bytes: an array of pointers */ + void *p[2]; +}; + +struct prog_test_ret_arr_struct { /* 16 bytes: the pointer is under an array of structs */ + struct { + struct { + void *p; + } in2; + } in1[2]; +}; + +struct prog_test_ret_arr2d { /* 16 bytes: a two dimensional array of structs */ + struct { + void *p; + } a[1][2]; +}; + +struct prog_test_ret_deep { /* 8 bytes, but nested past the 4-level walk limit */ + struct { + struct { + struct { + struct { + __u64 v; + } l4; + } l3; + } l2; + } l1; +}; + +struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ + __u64 a; + __u64 b; + __u64 c; +}; + struct prog_test_fail1 { void *p; int x; @@ -124,6 +224,8 @@ int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_nullable(void) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; @@ -131,6 +233,47 @@ int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym; struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym; long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; +#ifdef __SIZEOF_INT128__ +__int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; +#endif +struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; +struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; +struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +__u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym; +#ifdef __SIZEOF_INT128__ +__u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd(__u64 a, __int128 v, __u64 b) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd_many(__u64 a, __int128 v, __u64 b, __u64 c, + __u64 d, __u64 e, __u64 f, __int128 w) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_stack_hole(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct prog_test_pair_arg s, + __u64 f, __u64 g, + struct prog_test_i128_arg w) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __int128 v) __ksym; +#endif +__u64 bpf_kfunc_call_test_pair_arg_nofit(__u64 a, __u64 b, __u64 c, __u64 d, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_tail(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + struct prog_test_pair_arg s, __u64 f) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_split8(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arena_arg(__u64 a, __u64 b, __u64 c, + struct prog_test_pair_arg s, + __u64 *f__arena) __ksym; +struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; +struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; +struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; +struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(__u64 addr) __ksym; +struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(__u64 addr) __ksym; +struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(__u64 addr) __ksym; +union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(__u64 addr) __ksym; +struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; +struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym; @@ -161,6 +304,14 @@ __u64 bpf_kfunc_call_stack_arg_big(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) __ksym; void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym; void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym; +void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) __ksym; +void bpf_kfunc_test_uninit_mem(void *out__uninit, __u32 out__sz) __ksym; +void bpf_kfunc_test_uninit_partial(void *out__uninit, __u32 out__sz) __ksym; +int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) __ksym; +void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, __u32 b__sz) __ksym; +void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) __ksym; +void bpf_kfunc_test_uninit_stack(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + int *out__uninit) __ksym; void bpf_kfunc_call_test_mem_len_fail2(__u64 *mem, int len) __ksym; void bpf_kfunc_call_test_destructive(void) __ksym; @@ -204,6 +355,8 @@ int bpf_kfunc_st_ops_inc10(struct st_ops_args *args) __ksym; void bpf_kfunc_trusted_vma_test(struct vm_area_struct *ptr) __ksym; void bpf_kfunc_trusted_task_test(struct task_struct *ptr) __ksym; +struct task_struct *bpf_kfunc_trusted_null_test(void) __ksym; +void bpf_kfunc_trusted_nullable_test(struct task_struct *ptr__nullable) __ksym; void bpf_kfunc_trusted_num_test(int *ptr) __ksym; void bpf_kfunc_rcu_task_test(struct task_struct *ptr) __ksym; struct task_struct *bpf_kfunc_ret_rcu_test(void) __ksym;
diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh deleted file mode 100755 index 5252b91..0000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2.sh +++ /dev/null
@@ -1,41 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 -ret=$ksft_skip - -msg="skip all tests:" -if [ $UID != 0 ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -GREEN='\033[0;92m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -modprobe rc-loopback - -for i in /sys/class/rc/rc* -do - if grep -q DRV_NAME=rc-loopback $i/uevent - then - LIRCDEV=$(grep DEVNAME= $i/lirc*/uevent | sed sQDEVNAME=Q/dev/Q) - INPUTDEV=$(grep DEVNAME= $i/input*/event*/uevent | sed sQDEVNAME=Q/dev/Q) - fi -done - -if [ -n "$LIRCDEV" ]; -then - TYPE=lirc_mode2 - ./test_lirc_mode2_user $LIRCDEV $INPUTDEV - ret=$? - if [ $ret -ne 0 ]; then - echo -e ${RED}"FAIL: $TYPE"${NC} - else - echo -e ${GREEN}"PASS: $TYPE"${NC} - fi -fi - -exit $ret
diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c deleted file mode 100644 index cd191da..0000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ /dev/null
@@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young <sean@mess.org> - -// A lirc chardev is a device representing a consumer IR (cir) device which -// can receive infrared signals from remote control and/or transmit IR. -// -// IR is sent as a series of pulses and space somewhat like morse code. The -// BPF program can decode this into scancodes so that rc-core can translate -// this into input key codes using the rc keymap. -// -// This test works by sending IR over rc-loopback, so the IR is processed by -// BPF and then decoded into scancodes. The lirc chardev must be the one -// associated with rc-loopback, see the output of ir-keytable(1). -// -// The following CONFIG options must be enabled for the test to succeed: -// CONFIG_RC_CORE=y -// CONFIG_BPF_RAWIR_EVENT=y -// CONFIG_RC_LOOPBACK=y - -// Steps: -// 1. Open the /dev/lircN device for rc-loopback (given on command line) -// 2. Attach bpf_lirc_mode2 program which decodes some IR. -// 3. Send some IR to the same IR device; since it is loopback, this will -// end up in the bpf program -// 4. bpf program should decode IR and report keycode -// 5. We can read keycode from same /dev/lirc device - -#include <linux/bpf.h> -#include <linux/input.h> -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <poll.h> -#include <sys/types.h> -#include <sys/ioctl.h> -#include <sys/stat.h> -#include <fcntl.h> - -#include "bpf_util.h" -#include <bpf/bpf.h> -#include <bpf/libbpf.h> - -#include "testing_helpers.h" - -int main(int argc, char **argv) -{ - struct bpf_object *obj; - int ret, lircfd, progfd, inputfd; - int testir1 = 0x1ead; - int testir2 = 0x2101; - u32 prog_ids[10], prog_flags[10], prog_cnt; - - if (argc != 3) { - printf("Usage: %s /dev/lircN /dev/input/eventM\n", argv[0]); - return 2; - } - - ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o", - BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); - if (ret) { - printf("Failed to load bpf program\n"); - return 1; - } - - lircfd = open(argv[1], O_RDWR | O_NONBLOCK); - if (lircfd == -1) { - printf("failed to open lirc device %s: %m\n", argv[1]); - return 1; - } - - /* Let's try detach it before it was ever attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret != -ENOENT) { - printf("bpf_prog_detach2 not attached should fail: %m\n"); - return 1; - } - - inputfd = open(argv[2], O_RDONLY | O_NONBLOCK); - if (inputfd == -1) { - printf("failed to open input device %s: %m\n", argv[1]); - return 1; - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 0) { - printf("Expected nothing to be attached\n"); - return 1; - } - - ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); - if (ret) { - printf("Failed to attach bpf to lirc device: %m\n"); - return 1; - } - - /* Write raw IR */ - ret = write(lircfd, &testir1, sizeof(testir1)); - if (ret != sizeof(testir1)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - struct pollfd pfd = { .fd = inputfd, .events = POLLIN }; - struct input_event event; - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_MSC && event.code == MSC_SCAN && - event.value == 0x1ead) { - break; - } - } - - /* Write raw IR */ - ret = write(lircfd, &testir2, sizeof(testir2)); - if (ret != sizeof(testir2)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_REL && event.code == REL_Y && - event.value == 1 ) { - break; - } - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 1) { - printf("Expected one program to be attached\n"); - return 1; - } - - /* Let's try detaching it now it is actually attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret) { - printf("bpf_prog_detach2: returned %m\n"); - return 1; - } - - return 0; -}
diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 0780775..824e591 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c
@@ -33,6 +33,7 @@ static inline const char *str_has_pfx(const char *str, const char *pfx) #endif static int sysctl_unpriv_disabled = -1; +static int unpriv_mitigations_disabled = -1; enum mode { PRIV = 1, @@ -44,6 +45,11 @@ enum load_mode { NO_JITED = 1 << 1, }; +enum stack_mode { + LARGE_STACK = 1 << 0, + SMALL_STACK = 1 << 1, +}; + struct test_subspec { char *name; char *description; @@ -69,7 +75,10 @@ struct test_spec { int mode_mask; int arch_mask; int load_mask; + int stack_mask; int linear_sz; + const char *skip_reason; + bool prepare_priv; bool auxiliary; bool valid; }; @@ -422,6 +431,7 @@ static int parse_test_spec(struct test_loader *tester, int err = 0; u32 arch_mask = 0; u32 load_mask = 0; + u32 stack_mask = 0; struct btf *btf; enum arch arch; @@ -456,6 +466,8 @@ static int parse_test_spec(struct test_loader *tester, continue; if ((val = str_has_pfx(s, "test_description="))) { description = val; + } else if ((val = str_has_pfx(s, "test_skip="))) { + spec->skip_reason = val; } else if (strcmp(s, "test_expect_failure") == 0) { spec->priv.expect_failure = true; spec->mode_mask |= PRIV; @@ -496,6 +508,16 @@ static int parse_test_spec(struct test_loader *tester, if (err) goto cleanup; spec->mode_mask |= UNPRIV; + } else if ((msg = str_has_pfx(s, "test_expect_msg_next="))) { + err = __push_msg(msg, true, false, &spec->priv.expect_msgs); + if (err) + goto cleanup; + spec->mode_mask |= PRIV; + } else if ((msg = str_has_pfx(s, "test_expect_msg_next_unpriv="))) { + err = __push_msg(msg, true, false, &spec->unpriv.expect_msgs); + if (err) + goto cleanup; + spec->mode_mask |= UNPRIV; } else if ((msg = str_has_pfx(s, "test_jited="))) { if (arch_mask == 0) { PRINT_FAIL("__jited used before __arch_*"); @@ -568,6 +590,8 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); + } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { + update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) @@ -603,6 +627,8 @@ static int parse_test_spec(struct test_loader *tester, if (err) goto cleanup; spec->mode_mask |= UNPRIV; + } else if (strcmp(s, "test_prepare_priv") == 0) { + spec->prepare_priv = true; } else if ((val = str_has_pfx(s, "load_mode="))) { if (strcmp(val, "jited") == 0) { load_mask = JITED; @@ -613,6 +639,16 @@ static int parse_test_spec(struct test_loader *tester, err = -EINVAL; goto cleanup; } + } else if ((val = str_has_pfx(s, "stack_mode="))) { + if (strcmp(val, "large") == 0) { + stack_mask = LARGE_STACK; + } else if (strcmp(val, "small") == 0) { + stack_mask = SMALL_STACK; + } else { + PRINT_FAIL("bad stack spec: '%s'", val); + err = -EINVAL; + goto cleanup; + } } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) { err = push_disasm_msg(msg, &stderr_on_next_line, &spec->priv.stderr); @@ -652,6 +688,7 @@ static int parse_test_spec(struct test_loader *tester, spec->arch_mask = arch_mask ?: -1; spec->load_mask = load_mask ?: (JITED | NO_JITED); + spec->stack_mask = stack_mask ?: (LARGE_STACK | SMALL_STACK); if (spec->mode_mask == 0) spec->mode_mask = PRIV; @@ -745,7 +782,7 @@ static void prepare_case(struct test_loader *tester, struct bpf_object *obj, struct bpf_program *prog) { - int min_log_level = 0, prog_flags; + int min_log_level = 0; if (env.verbosity > VERBOSE_NONE) min_log_level = 1; @@ -763,8 +800,7 @@ static void prepare_case(struct test_loader *tester, else bpf_program__set_log_level(prog, spec->log_level); - prog_flags = bpf_program__flags(prog); - bpf_program__set_flags(prog, prog_flags | spec->prog_flags); + bpf_program__add_flags(prog, spec->prog_flags); tester->log_buf[0] = '\0'; } @@ -972,6 +1008,7 @@ void validate_msgs(const char *log_buf, struct expected_msgs *msgs, no_match = !msg->negative && !match->start; wrong_line = !msg->negative && + match->start && msg->on_next_line && prev_match && prev_match->line + 1 != match->line; unexpected = msg->negative && match->start; @@ -1013,10 +1050,10 @@ struct cap_state { bool initialized; }; -static int drop_capabilities(struct cap_state *caps) +static int drop_capabilities(struct cap_state *caps, __u64 keep_caps) { const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | - 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); + 1ULL << CAP_PERFMON | 1ULL << CAP_BPF) & ~keep_caps; int err; err = cap_disable_effective(caps_to_drop, &caps->old_caps); @@ -1026,6 +1063,13 @@ static int drop_capabilities(struct cap_state *caps) } caps->initialized = true; + if (keep_caps) { + err = cap_enable_effective(keep_caps, NULL); + if (err) { + PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); + return err; + } + } return 0; } @@ -1046,8 +1090,12 @@ static int restore_capabilities(struct cap_state *caps) static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) { if (sysctl_unpriv_disabled < 0) - sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; - if (sysctl_unpriv_disabled) + sysctl_unpriv_disabled = get_unpriv_sysctl_disabled(); + if (sysctl_unpriv_disabled && !(spec->unpriv.caps & (1ULL << CAP_BPF))) + return false; + if (unpriv_mitigations_disabled < 0) + unpriv_mitigations_disabled = get_unpriv_mitigations_disabled(); + if (unpriv_mitigations_disabled) return false; if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) return false; @@ -1314,6 +1362,7 @@ void run_subtest(struct test_loader *tester, { struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; int current_runtime = is_jit_enabled() ? JITED : NO_JITED; + int current_stack = is_large_stack_supported() ? LARGE_STACK : SMALL_STACK; struct bpf_program *tprog = NULL, *tprog_iter; struct bpf_link *link, *links[32] = {}; struct test_spec *spec_iter; @@ -1327,6 +1376,12 @@ void run_subtest(struct test_loader *tester, if (!test__start_subtest_with_desc(subspec->name, subspec->description)) return; + if (spec->skip_reason) { + printf("%s:SKIP: %s\n", __func__, spec->skip_reason); + test__skip(); + return; + } + if ((get_current_arch() & spec->arch_mask) == 0) { test__skip(); return; @@ -1337,23 +1392,19 @@ void run_subtest(struct test_loader *tester, return; } + if ((current_stack & spec->stack_mask) == 0) { + test__skip(); + return; + } + if (unpriv) { if (!can_execute_unpriv(tester, spec)) { test__skip(); test__end_subtest(); return; } - if (drop_capabilities(&caps)) { - test__end_subtest(); - return; - } - if (subspec->caps) { - err = cap_enable_effective(subspec->caps, NULL); - if (err) { - PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); - goto subtest_cleanup; - } - } + if (!spec->prepare_priv && drop_capabilities(&caps, subspec->caps)) + goto subtest_cleanup; } /* Implicitly reset to NULL if next test case doesn't specify. @@ -1406,6 +1457,18 @@ void run_subtest(struct test_loader *tester, bpf_object__for_each_map(map, tobj) bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); + if (unpriv && spec->prepare_priv) { + /* + * Module BTF lookup needs CAP_SYS_ADMIN. Allow tests to prepare + * their objects first, then verify programs with the requested caps. + */ + err = bpf_object__prepare(tobj); + if (!ASSERT_OK(err, "obj_prepare")) + goto tobj_cleanup; + if (drop_capabilities(&caps, subspec->caps)) + goto tobj_cleanup; + } + err = bpf_object__load(tobj); if (subspec->expect_failure) { if (!ASSERT_ERR(err, "unexpected_load_success")) { @@ -1483,7 +1546,8 @@ void run_subtest(struct test_loader *tester, } err = do_prog_test_run(bpf_program__fd(tprog), &retval, - bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false, + bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL || + bpf_program__type(tprog) == BPF_PROG_TYPE_RAW_TRACEPOINT, spec->linear_sz); if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) { PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 46eb201..168d84c 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c
@@ -2065,8 +2065,11 @@ int main(int argc, char **argv) libbpf_set_print(libbpf_print_fn); err = register_session_key((const char *)test_progs_verification_cert, test_progs_verification_cert_len); - if (err < 0) + if (err < 0) { + fprintf(stderr, "Failed to add libbpf_session_key to the session keyring: %s\n", + strerror(errno)); return err; + } traffic_monitor_set_print(traffic_monitor_print_fn);
diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index ea493c4..adc18f5 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h
@@ -349,6 +349,9 @@ void hexdump(const char *prefix, const void *buf, size_t len); ___ok; \ }) +#define ASSERT_TEXT_EQ(actual, expected, name) \ + ASSERT_TRUE(compare_text_to_expected(actual, expected) == 0, name) + #define ASSERT_MEMEQ(actual, expected, len, name) ({ \ static int duration = 0; \ const void *__act = actual; \
diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 9f672d0..1e3e31a 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c
@@ -8,6 +8,7 @@ #include <sys/mman.h> #include <alloca.h> #include <bpf/bpf.h> +#include <bpf/btf.h> #include <bpf/libbpf.h> #include "disasm.h" #include "test_progs.h" @@ -293,7 +294,6 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, ); struct bpf_object *obj; struct bpf_program *prog; - __u32 flags; int err; obj = bpf_object__open_file(file, &opts); @@ -309,8 +309,7 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) bpf_program__set_type(prog, type); - flags = bpf_program__flags(prog) | testing_prog_flags(); - bpf_program__set_flags(prog, flags); + bpf_program__add_flags(prog, testing_prog_flags()); err = bpf_object__load(obj); if (err) @@ -547,6 +546,47 @@ bool is_jit_enabled(void) return enabled; } +/* + * Whether the kernel accepts a program using more than 512 bytes of stack, + * which depends on the JIT in use. Probed once with a program that stores + * at the 2 KiB depth. Only the verifier's verdict on that store is cached: + * a load that fails for another reason, such as a missing capability, is + * reported and probed again on the next call. + */ +bool is_large_stack_supported(void) +{ + static int supported = -1; + struct bpf_insn insns[] = { + BPF_ST_MEM(BPF_DW, BPF_REG_10, -2048, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char log[1024] = {}; + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log, + .log_size = sizeof(log), + .log_level = 1, + ); + int fd; + + if (supported >= 0) + return supported; + + fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, ARRAY_SIZE(insns), + &opts); + if (fd >= 0) { + close(fd); + supported = 1; + } else if (strstr(log, "invalid write to stack")) { + supported = 0; + } else { + fprintf(stderr, "%s: probe failed with errno %d, assuming 512 bytes:\n%s", + __func__, errno, log); + return false; + } + return supported; +} + int stack_mprotect(void) { void *buf; @@ -562,3 +602,90 @@ int stack_mprotect(void) PROT_READ | PROT_WRITE | PROT_EXEC); return ret; } + +int compare_text_to_expected(const char *actual, const char *expected) +{ + char exp_path[] = "/tmp/selftest_expected.XXXXXX"; + char act_path[] = "/tmp/selftest_actual.XXXXXX"; + char buf[512], cmd[128]; + int exp_fd, act_fd; + FILE *p; + + if (!strcmp(actual, expected)) + return 0; + + exp_fd = mkstemp(exp_path); + act_fd = mkstemp(act_path); + if (exp_fd < 0 || act_fd < 0) { + fprintf(stdout, "output differs, no temp file for a diff\n"); + goto out; + } + + dprintf(exp_fd, "%s", expected); + dprintf(act_fd, "%s", actual); + + snprintf(cmd, sizeof(cmd), "diff -u '%s' '%s'", exp_path, act_path); + p = popen(cmd, "r"); + if (!p) { + fprintf(stdout, "output differs, '%s' did not run\n", cmd); + goto out; + } + while (fgets(buf, sizeof(buf), p)) + fputs(buf, stdout); + pclose(p); + +out: + if (exp_fd >= 0) { + close(exp_fd); + unlink(exp_path); + } + if (act_fd >= 0) { + close(act_fd); + unlink(act_path); + } + return -1; +} + +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_member *m; + const struct btf_type *t; + int err = 0, id, i; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return -errno; + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + err = -errno; + goto out; + } + id = btf__find_by_name_kind_own(module_btf, struct_name, BTF_KIND_STRUCT); + if (id < 0) { + err = id; + goto out; + } + t = btf__type_by_id(module_btf, id); + err = -ENOENT; + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + if (strcmp(btf__name_by_offset(module_btf, m->name_off), member)) + continue; + err = 0; + /* the tag sits on the pointee, behind the pointer and other modifiers */ + for (id = m->type; id; id = t->type) { + t = btf__type_by_id(module_btf, id); + if (btf_is_type_tag(t) && !strcmp(btf__name_by_offset(module_btf, t->name_off), tag)) { + err = 1; + break; + } + if (!btf_is_ptr(t) && !btf_is_mod(t) && !btf_is_typedef(t)) + break; + } + break; + } +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + return err; +}
diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 1f83b60..36a7515 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h
@@ -61,6 +61,17 @@ struct bpf_insn; int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt); int testing_prog_flags(void); bool is_jit_enabled(void); +bool is_large_stack_supported(void); int stack_mprotect(void); +/* Runs diff(1) on mismatch */ +int compare_text_to_expected(const char *actual, const char *expected); + +/* + * Whether @member of struct @struct_name in the BTF of bpf_testmod carries + * the BTF type tag @tag: 1 if so, 0 if not, negative on a BTF error or when + * the struct or the member is missing. + */ +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag); + #endif /* __TESTING_HELPERS_H */
diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index 679008b..6d7a4b8 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c
@@ -244,6 +244,45 @@ int kallsyms_find(const char *sym, unsigned long long *addr) return err; } +unsigned long long module_get_base_addr(const char *module) +{ + unsigned long long addr, base_addr = 0; + char name[500]; + FILE *f; + + f = fopen("/proc/modules", "r"); + if (!f) + return 0; + + while (fscanf(f, "%499s %*s %*s %*s %*s 0x%llx", name, &addr) == 2) { + if (strcmp(module, name) == 0) { + base_addr = addr; + break; + } + } + fclose(f); + return base_addr; +} + +unsigned long long module_get_base_addr_sec(const char *module, const char *sec) +{ + unsigned long long addr = 0; + char *path; + FILE *f; + + if (asprintf(&path, "/sys/module/%s/sections/%s", module, sec) < 0) + return 0; + f = fopen(path, "r"); + free(path); + if (!f) + return 0; + + if (fscanf(f, "%llx", &addr) != 1) + addr = 0; + fclose(f); + return addr; +} + #ifdef PROCMAP_QUERY int env_verbosity __weak = 0;
diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h index 01c8ecc..0f8d997 100644 --- a/tools/testing/selftests/bpf/trace_helpers.h +++ b/tools/testing/selftests/bpf/trace_helpers.h
@@ -38,6 +38,8 @@ typedef int (*ksym_search_cmp_t)(const void *p1, const struct ksym *p2); int load_kallsyms(void); struct ksym *ksym_search(long key); long ksym_get_addr(const char *name); +unsigned long long module_get_base_addr(const char *module); +unsigned long long module_get_base_addr_sec(const char *module, const char *sec); struct ksyms *load_kallsyms_local(void); struct ksym *ksym_search_local(struct ksyms *ksyms, long key);
diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index f997d7e..c8dd5d8 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c
@@ -72,8 +72,8 @@ static int config_contains(const char *pat) static bool cmdline_contains(const char *pat) { + int fd, cnt, ret = false; char cmdline[4096], *c; - int fd, ret = false; fd = open("/proc/cmdline", O_RDONLY); if (fd < 0) { @@ -81,14 +81,15 @@ static bool cmdline_contains(const char *pat) return false; } - if (read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + cnt = read(fd, cmdline, sizeof(cmdline) - 1); + if (cnt < 0) { perror("read /proc/cmdline"); goto out; } - cmdline[sizeof(cmdline) - 1] = '\0'; + cmdline[cnt] = '\0'; for (c = strtok(cmdline, " \n"); c; c = strtok(NULL, " \n")) { - if (strncmp(c, pat, strlen(c))) + if (strcmp(c, pat)) continue; ret = true; break; @@ -110,9 +111,8 @@ static int get_mitigations_off(void) return !enabled_in_config; } -bool get_unpriv_disabled(void) +bool get_unpriv_sysctl_disabled(void) { - int mitigations_off; bool disabled; char buf[2]; FILE *fd; @@ -126,8 +126,12 @@ bool get_unpriv_disabled(void) disabled = true; } - if (disabled) - return true; + return disabled; +} + +bool get_unpriv_mitigations_disabled(void) +{ + int mitigations_off; /* * Some unpriv tests rely on spectre mitigations being on. @@ -142,3 +146,18 @@ bool get_unpriv_disabled(void) } return mitigations_off; } + +bool get_unpriv_disabled(void) +{ + return get_unpriv_sysctl_disabled() || get_unpriv_mitigations_disabled(); +} + +bool get_kasan_jit_enabled(void) +{ + return config_contains("CONFIG_BPF_JIT_KASAN=y") == 1; +} + +bool get_kasan_multi_shot_enabled(void) +{ + return cmdline_contains("kasan_multi_shot"); +}
diff --git a/tools/testing/selftests/bpf/unpriv_helpers.h b/tools/testing/selftests/bpf/unpriv_helpers.h index 151f673..c24d53e 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.h +++ b/tools/testing/selftests/bpf/unpriv_helpers.h
@@ -5,3 +5,7 @@ #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" bool get_unpriv_disabled(void); +bool get_unpriv_sysctl_disabled(void); +bool get_unpriv_mitigations_disabled(void); +bool get_kasan_jit_enabled(void); +bool get_kasan_multi_shot_enabled(void);
diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 5e38f86..3b7024b 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c
@@ -13,6 +13,22 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +/* + * Force the nop1,nop10 combo of the USDT probe to a spot where the nop10 + * crosses a page boundary: .balign starts the padding exactly at a page + * start regardless of the compiler-generated prologue size, and the 4086 + * one-byte nops put the nop1 at page offset 4086, so the following nop10 + * occupies the last 9 bytes of that page and 1 byte of the next one. + * The kernel can't optimize such nop10, so libbpf must keep the uprobe + * on the 1-byte nop. + */ +__attribute__((noinline)) +void usdt_2_cross_page(void) +{ + asm volatile (".balign 4096, 0x90\n\t.skip 4086, 0x90"); + USDT(optimized_attach, usdt_2_cross_page); +} + static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE;
diff --git a/tools/testing/selftests/bpf/verifier/basic_call.c b/tools/testing/selftests/bpf/verifier/basic_call.c index a8c6ab4..0f93c45 100644 --- a/tools/testing/selftests/bpf/verifier/basic_call.c +++ b/tools/testing/selftests/bpf/verifier/basic_call.c
@@ -4,7 +4,7 @@ BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, 0, 0, 0, 0), BPF_EXIT_INSN(), }, - .errstr = "unknown opcode 8d", + .errstr = "R0 !read_ok", .result = REJECT, }, {
diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index eb6e3ba..0af237c 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c
@@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 is fp expected STRUCT prog_test_fail1", + .errstr = "R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 is fp expected STRUCT prog_test_fail3", + .errstr = "R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, }, @@ -76,7 +76,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 expected pointer to ctx, but got fp", + .errstr = "R1 type=fp expected=ctx", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_pass_ctx", 2 }, }, @@ -152,7 +152,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "kernel function bpf_kfunc_call_memb1_release R1 expected pointer", + .errstr = "bpf_kfunc_call_memb1_release R1 expected pointer", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_memb_acquire", 1 }, { "bpf_kfunc_call_memb1_release", 5 }, @@ -1037,15 +1037,34 @@ .result = ACCEPT, }, { - "calls: stack overflow using two frames (pre-call access)", + /* + * Five 480-byte frames exceed the 2 KiB budget of JITs with large + * stacks, and two of them the 512 bytes allowed elsewhere. + */ + "calls: stack overflow using five frames (pre-call access)", .insns = { /* prog 1 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), BPF_EXIT_INSN(), /* prog 2 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 3 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 4 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 5 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1054,15 +1073,30 @@ .result = REJECT, }, { - "calls: stack overflow using two frames (post-call access)", + "calls: stack overflow using five frames (post-call access)", .insns = { /* prog 1 */ BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), /* prog 2 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 3 */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 4 */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 5 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1127,7 +1161,7 @@ .result = ACCEPT, }, { - "calls: stack depth check using three frames. test3", + "calls: stack depth check using five frames. test3", .insns = { /* main */ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), @@ -1135,66 +1169,104 @@ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 8), /* call B */ BPF_JMP_IMM(BPF_JGE, BPF_REG_6, 0, 1), - BPF_ST_MEM(BPF_B, BPF_REG_10, -64, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), /* A */ BPF_JMP_IMM(BPF_JLT, BPF_REG_1, 10, 1), BPF_EXIT_INSN(), - BPF_ST_MEM(BPF_B, BPF_REG_10, -224, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_JMP_IMM(BPF_JA, 0, 0, -3), /* B */ BPF_JMP_IMM(BPF_JGT, BPF_REG_1, 2, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -6), /* call A */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -256, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), /* call C */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + /* C */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), /* call D */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + /* D */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -12), /* call A */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_XDP, - /* stack_main=64, stack_A=224, stack_B=256 - * and max(main+A, main+A+B) > 512 + /* + * every frame is 480 bytes, main+A = 960 > 512 and + * max(main+A, main+B+C+D+A) = 2400 > 2048 */ .errstr = "combined stack", .result = REJECT, }, { - "calls: stack depth check using three frames. test4", - /* void main(void) { + "calls: stack depth check using five frames. test4", + /* + * void main(void) { * func1(0); * func1(1); * func2(1); * } - * void func1(int alloc_or_recurse) { + * void funcN(int alloc_or_recurse) { N = 1..4 * if (alloc_or_recurse) { - * frame_pointer[-300] = 1; + * frame_pointer[-480] = 1; * } else { - * func2(alloc_or_recurse); + * funcN+1(alloc_or_recurse); * } * } - * void func2(int alloc_or_recurse) { + * void func5(int alloc_or_recurse) { * if (alloc_or_recurse) { - * frame_pointer[-300] = 1; + * frame_pointer[-480] = 1; * } * } + * main also calls func2 to func5 with 1 so that every function has a + * path allocating its 480 bytes, and the chain adds up to 2400 bytes, + * more than the 2 KiB budget of JITs with large stacks, and to 960 + * bytes after two frames, more than the 512 bytes allowed elsewhere. */ .insns = { /* main */ BPF_MOV64_IMM(BPF_REG_1, 0), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 6), /* call A */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 12), /* call A */ BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4), /* call A */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 10), /* call A */ BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 7), /* call B */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 13), /* call B */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 16), /* call C */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 19), /* call D */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 22), /* call E */ BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), /* A */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call B */ BPF_EXIT_INSN(), /* B */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call C */ + BPF_EXIT_INSN(), + /* C */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call D */ + BPF_EXIT_INSN(), + /* D */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call E */ + BPF_EXIT_INSN(), + /* E */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 1), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_XDP,
diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c index 4b39f84..345cecc 100644 --- a/tools/testing/selftests/bpf/verifier/map_kptr.c +++ b/tools/testing/selftests/bpf/verifier/map_kptr.c
@@ -311,7 +311,7 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_kptr = { 1 }, .result = REJECT, - .errstr = "R1 type=rcu_ptr_or_null_ expected=percpu_ptr_", + .errstr = "Possibly NULL pointer passed to trusted R1", }, { "map_kptr: ref: reject off != 0", @@ -342,7 +342,7 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_kptr = { 1 }, .result = REJECT, - .errstr = "invalid kptr access, R2 type=ptr_prog_test_ref_kfunc expected=ptr_prog_test_member", + .errstr = "R2 must have zero offset when passed to release func", }, { "map_kptr: ref: reference state created and released on xchg",
diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index 09179fb5..ba5921e 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh
@@ -28,7 +28,7 @@ usage() { - echo "Usage: $0 <setup|cleanup <existing_tmp_dir>" + echo "Usage: $0 <setup-rsa|setup-mldsa|cleanup <existing_tmp_dir>" exit 1 } @@ -47,7 +47,7 @@ ${tmp_dir}/signing_key.der -outform der } -setup() +setup_rsa() { local tmp_dir="$1" @@ -57,6 +57,57 @@ keyctl link $key_id $keyring_id } +mldsa_supported() +{ + local tmp_dir="$1" + + genkey_mldsa "${tmp_dir}" || return 1 + : > ${tmp_dir}/probe + # Same digest as the caller signs with, see sign_buf_digest(). + ./sign-file -d sha512 ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.pem ${tmp_dir}/probe || return 1 + rm -f ${tmp_dir}/probe ${tmp_dir}/probe.p7s +} + +genkey_mldsa() +{ + local tmp_dir="$1" + + echo "${x509_genkey_content}" > ${tmp_dir}/x509.genkey + + # No -<digest> here: ML-DSA hashes the message itself, so openssl + # ignores an explicit digest for it. + openssl req -new -nodes -utf8 -days 36500 \ + -batch -x509 -newkey ML-DSA-87 \ + -config ${tmp_dir}/x509.genkey \ + -outform PEM -out ${tmp_dir}/signing_key.pem \ + -keyout ${tmp_dir}/signing_key.pem 2>&1 + + openssl x509 -in ${tmp_dir}/signing_key.pem -out \ + ${tmp_dir}/signing_key.der -outform der +} + +mldsa_skip() +{ + local tmp_dir="$1" + + rm -f ${tmp_dir}/x509.genkey ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.der ${tmp_dir}/probe \ + ${tmp_dir}/probe.p7s + exit 77 +} + +setup_mldsa() +{ + local tmp_dir="$1" + + mldsa_supported "${tmp_dir}" || mldsa_skip "${tmp_dir}" + key_id=$(cat ${tmp_dir}/signing_key.der | + keyctl padd asymmetric ebpf_testing_key @s) + keyring_id=$(keyctl newring ebpf_testing_keyring @s) + keyctl link $key_id $keyring_id +} + cleanup() { local tmp_dir="$1" @@ -91,7 +142,7 @@ local exit_code="$1" local log_file="$2" - if [[ "${exit_code}" -ne 0 ]]; then + if [[ "${exit_code}" -ne 0 && "${exit_code}" -ne 77 ]]; then cat "${log_file}" >&3 fi @@ -108,8 +159,10 @@ [[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1 - if [[ "${action}" == "setup" ]]; then - setup "${tmp_dir}" + if [[ "${action}" == "setup-rsa" ]]; then + setup_rsa "${tmp_dir}" + elif [[ "${action}" == "setup-mldsa" ]]; then + setup_mldsa "${tmp_dir}" elif [[ "${action}" == "genkey" ]]; then genkey "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then
diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e70741c..9cfc9b4 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c
@@ -1722,9 +1722,9 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf fixup_obj(obj, prog, base_filename); if (env.force_checkpoints) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ); + bpf_program__add_flags(prog, BPF_F_TEST_STATE_FREQ); if (env.force_reg_invariants) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); + bpf_program__add_flags(prog, BPF_F_TEST_REG_INVARIANTS); opts.log_buf = buf; opts.log_size = buf_sz;
diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 6a3d026..c4663ca 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh
@@ -35,11 +35,11 @@ ARCH="arm64" ;; riscv64) - # required qemu version v7.2.0+ + # required qemu version v8.2.0+ QEMU_BINARY=qemu-system-riscv64 QEMU_CONSOLE="ttyS0,115200" HOST_FLAGS=(-M virt -cpu host -enable-kvm -smp 8) - CROSS_FLAGS=(-M virt -cpu rv64,sscofpmf=true -smp 8) + CROSS_FLAGS=(-M virt -cpu max,sscofpmf=false -smp 8) BZIMAGE="arch/riscv/boot/Image" ARCH="riscv" ;; @@ -65,7 +65,7 @@ KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config" "tools/testing/selftests/bpf/config.vm" "tools/testing/selftests/bpf/config.${PLATFORM}") -INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX" +INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/main/INDEX" NUM_COMPILE_JOBS="$(nproc)" LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")" LOG_FILE="${LOG_FILE_BASE}.log" @@ -107,6 +107,14 @@ -s) Instead of powering off the VM, start an interactive shell. If <command> is specified, the shell runs after the command finishes executing + +Environment variables: + + KERNEL_CMDLINE_EXTRA + Extra arguments to append to the guest kernel command + line, for tests that need a boot-time setting. e.g: + + KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" $0 -- ./test_progs -t signed_loader EOF } @@ -286,6 +294,12 @@ QEMU_FLAGS=("${HOST_FLAGS[@]}") fi + local kernel_cmdline="root=/dev/vda rw console=${QEMU_CONSOLE}" + + if [[ -n "${KERNEL_CMDLINE_EXTRA:-}" ]]; then + kernel_cmdline+=" ${KERNEL_CMDLINE_EXTRA}" + fi + ${QEMU_BINARY} \ -nodefaults \ -display none \ @@ -294,7 +308,7 @@ -m 4G \ -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \ -kernel "${kernel_bzimage}" \ - -append "root=/dev/vda rw console=${QEMU_CONSOLE}" + -append "${kernel_cmdline}" } copy_logs()
diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index 6db3b55..ab3f5fc5 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c
@@ -845,7 +845,7 @@ int main(int argc, char *argv[]) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); printf("load bpf program...\n"); ret = xdp_hw_metadata__load(bpf_obj);
diff --git a/tools/testing/selftests/ftrace/Makefile b/tools/testing/selftests/ftrace/Makefile index 7c12263..3d41f95 100644 --- a/tools/testing/selftests/ftrace/Makefile +++ b/tools/testing/selftests/ftrace/Makefile
@@ -2,8 +2,8 @@ all: TEST_PROGS_EXTENDED := ftracetest -TEST_PROGS := ftracetest-ktap -TEST_FILES := test.d settings +TEST_PROGS := ftracetest-ktap boottime-ktap +TEST_FILES := test.d settings boottime EXTRA_CLEAN := $(OUTPUT)/logs/* TEST_GEN_FILES := poll
diff --git a/tools/testing/selftests/ftrace/boottime-ktap b/tools/testing/selftests/ftrace/boottime-ktap new file mode 100755 index 0000000..1fed68c --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime-ktap
@@ -0,0 +1,6 @@ +#!/bin/sh -e +# SPDX-License-Identifier: GPL-2.0-only +# +# boottime-ktap: Wrapper to integrate boottime tracing test framework with kselftest runner + +exec ./boottime/run_boottime_test.sh "$@"
diff --git a/tools/testing/selftests/ftrace/boottime/Makefile b/tools/testing/selftests/ftrace/boottime/Makefile new file mode 100644 index 0000000..fd09753 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/Makefile
@@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 + +all: + +run_tests: + @./run_boottime_test.sh + +clean: + +.PHONY: all run_tests clean
diff --git a/tools/testing/selftests/ftrace/boottime/README b/tools/testing/selftests/ftrace/boottime/README new file mode 100644 index 0000000..1eb0191 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/README
@@ -0,0 +1,74 @@ +Boot Tracing Test Suite +======================= + +This directory contains an automated test framework to verify kernel boot +tracing configurations (bootconfig, kernel command-line tracing options, and +persistent ring buffers) during early boot. + +Overview +-------- +The test runner (`run_boottime_test.sh`) constructs a lightweight initramfs with +busybox and a tracefs checker script from `tests/`, attaches optional bootconfigs +from `bootconfigs/` using the `bootconfig` tool, applies optional command-line +parameters from `cmdlines/`, boots the target kernel image under QEMU, and reports +results according to TAP version 13 format. + +Test Specification Format +------------------------- +For each test script `tests/<name>.sh`, the harness dynamically detects: +- `bootconfigs/<name>.bconf` or `<name>.bconf`: Bootconfig configuration file. +- `cmdlines/<name>.cmdline` or `# CMDLINE: <opts>` header in script: Kernel boot parameters. +- `qemuopts/<name>.qemuopts` or `# QEMUOPTS: <opts>` header in script: Additional QEMU CLI arguments. +- `# APPLETS: <applets>` header in script: Additional BusyBox applet symlinks to create in initramfs. +- `# REBOOT: 1` header in script: Multi-boot reboot/crash test mode (omits `-no-reboot` and doubles timeout). + +Required Tools +-------------- +- Host Architecture: x86 (x86_64/i686/i386/x86; tests skip on non-x86 hosts) +- `qemu-system-x86_64` (or custom QEMU executable) +- `busybox` (host binary at `/usr/bin/busybox` or custom path) +- `bootconfig` (compiled from `tools/bootconfig/bootconfig` or in PATH) +- `file` (standard file architecture identifier) +- `cpio` (standard archiver) +- `timeout` (standard coreutils execution timer) + +Required Kernel Configurations +------------------------------ +The kernel binary under test must be compiled with: +- `CONFIG_BOOT_CONFIG=y` +- `CONFIG_BOOTTIME_TRACING=y` +- `CONFIG_MAGIC_SYSRQ=y` (for guest reboot and poweroff) + +Additional feature-specific config options enable respective testcases: +- `CONFIG_KPROBE_EVENTS=y` (for kprobe tests) +- `CONFIG_SYNTH_EVENTS=y` (for synthetic event tests) +- `CONFIG_EPROBE_EVENTS=y` (for eprobe tests) +- `CONFIG_FPROBE_EVENTS=y` (for fprobe and tracepoint probe tests) +- `CONFIG_FUNCTION_TRACER=y` (for tracer options) +- `CONFIG_RESERVE_MEM=y` (for persistent ring buffer tests) + +Usage +----- +To run all test cases with a compiled kernel image: + + $ ./run_boottime_test.sh -k /path/to/bzImage + +To run a specific test case (e.g. `01-kprobe`): + + $ ./run_boottime_test.sh -k /path/to/bzImage -t 01-kprobe + +Customizing Tool Paths: + + $ ./run_boottime_test.sh -b /path/to/bootconfig \ + -k /path/to/bzImage \ + -q /path/to/qemu-system-x86_64 \ + -B /path/to/busybox + +Environment Variables +--------------------- +Tool locations and kernel paths can also be set via environment variables: +- `BOOTCONFIG`: Path to the `bootconfig` tool +- `KERNEL`: Path to the `bzImage` kernel binary +- `QEMU`: Path to the QEMU binary +- `BUSYBOX`: Path to the `busybox` binary +- `LOGDIR`: Directory path to preserve QEMU execution log files
diff --git a/tools/testing/selftests/ftrace/boottime/bootconfigs/01-kprobe.bconf b/tools/testing/selftests/ftrace/boottime/bootconfigs/01-kprobe.bconf new file mode 100644 index 0000000..13d30d1 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/bootconfigs/01-kprobe.bconf
@@ -0,0 +1,4 @@ +ftrace.event.kprobes.vfs_read { + probes = "vfs_read $arg1" + enable +}
diff --git a/tools/testing/selftests/ftrace/boottime/bootconfigs/02-synth.bconf b/tools/testing/selftests/ftrace/boottime/bootconfigs/02-synth.bconf new file mode 100644 index 0000000..b1f3771 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/bootconfigs/02-synth.bconf
@@ -0,0 +1,4 @@ +ftrace.event.synthetic.boot_lat { + fields = "unsigned long id", "u64 delta" + enable +}
diff --git a/tools/testing/selftests/ftrace/boottime/bootconfigs/03-eprobe.bconf b/tools/testing/selftests/ftrace/boottime/bootconfigs/03-eprobe.bconf new file mode 100644 index 0000000..c1a8138 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/bootconfigs/03-eprobe.bconf
@@ -0,0 +1,4 @@ +ftrace.event.eprobes.ep_read { + probes = "sched/sched_switch" + enable +}
diff --git a/tools/testing/selftests/ftrace/boottime/bootconfigs/04-fprobe.bconf b/tools/testing/selftests/ftrace/boottime/bootconfigs/04-fprobe.bconf new file mode 100644 index 0000000..82ae2ac --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/bootconfigs/04-fprobe.bconf
@@ -0,0 +1,4 @@ +ftrace.event.fprobes.fp_read { + probes = "vfs_read" + enable +}
diff --git a/tools/testing/selftests/ftrace/boottime/bootconfigs/05-tprobe.bconf b/tools/testing/selftests/ftrace/boottime/bootconfigs/05-tprobe.bconf new file mode 100644 index 0000000..02a2803 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/bootconfigs/05-tprobe.bconf
@@ -0,0 +1,4 @@ +ftrace.event.tracepoints.tp_sched { + probes = "sched_switch" + enable +}
diff --git a/tools/testing/selftests/ftrace/boottime/bootconfigs/06-instance.bconf b/tools/testing/selftests/ftrace/boottime/bootconfigs/06-instance.bconf new file mode 100644 index 0000000..3fdb671 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/bootconfigs/06-instance.bconf
@@ -0,0 +1,5 @@ +ftrace.instance.foo { + event.sched.sched_switch { + enable + } +}
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-01-ftrace.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-01-ftrace.cmdline new file mode 100644 index 0000000..4f6fc54 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-01-ftrace.cmdline
@@ -0,0 +1 @@ +ftrace=function
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-02-trace-event.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-02-trace-event.cmdline new file mode 100644 index 0000000..66bcd29 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-02-trace-event.cmdline
@@ -0,0 +1 @@ +trace_event=sched:sched_switch,kmem:kmalloc
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-03-trace-buf-size.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-03-trace-buf-size.cmdline new file mode 100644 index 0000000..e5d1c4a --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-03-trace-buf-size.cmdline
@@ -0,0 +1 @@ +trace_buf_size=2048K
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-04-trace-options.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-04-trace-options.cmdline new file mode 100644 index 0000000..51d82f8 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-04-trace-options.cmdline
@@ -0,0 +1 @@ +trace_options=sym-addr,verbose
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-05-trace-clock.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-05-trace-clock.cmdline new file mode 100644 index 0000000..f72a4ac --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-05-trace-clock.cmdline
@@ -0,0 +1 @@ +trace_clock=global
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-06-trace-instance.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-06-trace-instance.cmdline new file mode 100644 index 0000000..9618e3e --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/cmdline-06-trace-instance.cmdline
@@ -0,0 +1 @@ +trace_instance=bar,sched:sched_switch
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/persistent-01-reserve-mem.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/persistent-01-reserve-mem.cmdline new file mode 100644 index 0000000..c69eeb2 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/persistent-01-reserve-mem.cmdline
@@ -0,0 +1 @@ +reserve_mem=12M:32M:trace trace_instance=boot_map@trace
diff --git a/tools/testing/selftests/ftrace/boottime/cmdlines/persistent-02-backup-instance.cmdline b/tools/testing/selftests/ftrace/boottime/cmdlines/persistent-02-backup-instance.cmdline new file mode 100644 index 0000000..43217c9 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/cmdlines/persistent-02-backup-instance.cmdline
@@ -0,0 +1 @@ +reserve_mem=12M:32M:trace trace_instance=boot_map@trace trace_instance=backup=boot_map
diff --git a/tools/testing/selftests/ftrace/boottime/run_boottime_test.sh b/tools/testing/selftests/ftrace/boottime/run_boottime_test.sh new file mode 100755 index 0000000..6e8593d --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/run_boottime_test.sh
@@ -0,0 +1,403 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# +# Generic Boot Tracing Test Harness +# Builds a lightweight initramfs, applies bootconfigs and kernel cmdline parameters, +# boots QEMU, and verifies tracefs configuration using test scripts. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ -d "$SCRIPT_DIR/boottime" ]; then + SCRIPT_DIR="$SCRIPT_DIR/boottime" +fi +KERNEL_SRC="$(cd "$SCRIPT_DIR/../../../../.." && pwd)" + +# Source KTAP helpers +if [ -f "$SCRIPT_DIR/../../kselftest/ktap_helpers.sh" ]; then + KSELFTEST_DIR="$(cd "$SCRIPT_DIR/../../kselftest" && pwd)" +elif [ -f "$SCRIPT_DIR/../kselftest/ktap_helpers.sh" ]; then + KSELFTEST_DIR="$(cd "$SCRIPT_DIR/../kselftest" && pwd)" +else + echo "Error: ktap_helpers.sh not found relative to $SCRIPT_DIR" >&2 + exit 1 +fi +. "$KSELFTEST_DIR/ktap_helpers.sh" + +BOOTCONFIG="${BOOTCONFIG:-"$KERNEL_SRC/tools/bootconfig/bootconfig"}" +KERNEL="${KERNEL:-"$KERNEL_SRC/arch/x86/boot/bzImage"}" +QEMU="${QEMU:-"qemu-system-x86_64"}" +BUSYBOX="${BUSYBOX:-"$(which busybox 2>/dev/null || echo "/usr/bin/busybox")"}" +TIMEOUT="${TIMEOUT:-60s}" +LOGDIR="${LOGDIR:-""}" +TARGET_TEST="all" + +BUSYBOX_APPLETS=( + sh + cat + mount + umount + echo + sync + reboot + poweroff + cpio + grep + sleep + sed +) + +usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " -b, --bootconfig PATH Path to bootconfig tool (default: $BOOTCONFIG)" + echo " -k, --kernel PATH Path to kernel image (default: $KERNEL)" + echo " -q, --qemu PATH Path to QEMU executable (default: $QEMU)" + echo " -B, --busybox PATH Path to busybox binary (default: $BUSYBOX)" + echo " -T, --timeout DURATION QEMU execution timeout (default: $TIMEOUT)" + echo " -l, --logdir DIR Directory to save QEMU log files" + echo " -t, --test NAME Run specific test (e.g. 01-kprobe) or 'all'" + echo " -h, --help Show this help message" + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + -b|--bootconfig) + BOOTCONFIG="$2" + shift 2 + ;; + -k|--kernel) + KERNEL="$2" + shift 2 + ;; + -q|--qemu) + QEMU="$2" + shift 2 + ;; + -B|--busybox) + BUSYBOX="$2" + shift 2 + ;; + -T|--timeout) + TIMEOUT="$2" + shift 2 + ;; + -l|--logdir) + LOGDIR="$2" + shift 2 + ;; + -t|--test) + TARGET_TEST="$2" + shift 2 + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +# Check host architecture +HOST_ARCH="$(uname -m)" +case "$HOST_ARCH" in + x86_64|i686|i386|x86) + ;; + *) + if [ "$KERNEL" = "$KERNEL_SRC/arch/x86/boot/bzImage" ]; then + ktap_print_header + ktap_skip_all \ + "Default boot tracing test is x86 only ($HOST_ARCH)" + exit 0 + fi + ;; +esac + +# Ensure bootconfig tool is built if Makefile exists +if [ ! -x "$BOOTCONFIG" ]; then + if [ -f "$KERNEL_SRC/tools/bootconfig/Makefile" ]; then + make -C "$KERNEL_SRC/tools/bootconfig" > /dev/null 2>&1 || true + fi + if [ ! -x "$BOOTCONFIG" ]; then + BOOTCONFIG="$(which bootconfig 2>/dev/null || true)" + fi +fi + +if [ ! -x "$BOOTCONFIG" ]; then + ktap_print_header + ktap_skip_all "bootconfig tool not found at $BOOTCONFIG" + exit 0 +fi + +if [ ! -f "$KERNEL" ]; then + ktap_print_header + ktap_skip_all "Kernel image not found at $KERNEL" + exit 0 +fi + +if ! command -v "$QEMU" >/dev/null 2>&1; then + ktap_print_header + ktap_skip_all "QEMU binary not found: $QEMU" + exit 0 +fi + +if [ ! -x "$BUSYBOX" ]; then + ktap_print_header + ktap_skip_all "busybox binary not found at $BUSYBOX" + exit 0 +fi + +# Verify file command existence and architecture compatibility +if ! command -v file >/dev/null 2>&1; then + ktap_print_header + ktap_skip_all "file tool not found" + exit 0 +fi + +KERNEL_INFO="$(file -bL "$KERNEL" 2>/dev/null || true)" +case "$KERNEL_INFO" in + *x86*) + ;; + *) + ktap_print_header + ktap_skip_all "Kernel architecture is not x86 ($KERNEL_INFO)" + exit 0 + ;; +esac + +BUSYBOX_INFO="$(file -bL "$BUSYBOX" 2>/dev/null || true)" +case "$BUSYBOX_INFO" in + *x86-64*|*x86_64*|*80386*|*i386*) + ;; + *) + ktap_print_header + ktap_skip_all \ + "busybox binary architecture is incompatible with x86 kernel ($BUSYBOX_INFO)" + exit 0 + ;; +esac + +if ! command -v cpio >/dev/null 2>&1; then + ktap_print_header + ktap_skip_all "cpio tool not found" + exit 0 +fi + +if ! command -v timeout >/dev/null 2>&1; then + ktap_print_header + ktap_skip_all "timeout tool not found" + exit 0 +fi + +double_timeout() { + local val="$1" + if [[ "$val" =~ ^([0-9]+)([a-z]*)$ ]]; then + local num="${BASH_REMATCH[1]}" + local unit="${BASH_REMATCH[2]}" + echo "$((num * 2))$unit" + else + echo "120s" + fi +} + +run_single_test() { + local test_script="$1" + local name + local dir + name="$(basename "$test_script" .sh)" + dir="$(dirname "$test_script")" + + local bconf_file="" + if [ -f "$SCRIPT_DIR/bootconfigs/$name.bconf" ]; then + bconf_file="$SCRIPT_DIR/bootconfigs/$name.bconf" + elif [ -f "$dir/$name.bconf" ]; then + bconf_file="$dir/$name.bconf" + fi + + local test_cmdline="" + if [ -f "$SCRIPT_DIR/cmdlines/$name.cmdline" ]; then + test_cmdline="$(cat "$SCRIPT_DIR/cmdlines/$name.cmdline")" + elif [ -f "$dir/$name.cmdline" ]; then + test_cmdline="$(cat "$dir/$name.cmdline")" + fi + + local inline_cmdline + inline_cmdline="$(grep -E '^# *CMDLINE:' "$test_script" | sed 's/^# *CMDLINE://' | xargs || true)" + if [ -n "$inline_cmdline" ]; then + test_cmdline="$test_cmdline $inline_cmdline" + fi + + local test_qemuopts="" + if [ -f "$SCRIPT_DIR/qemuopts/$name.qemuopts" ]; then + test_qemuopts="$(cat "$SCRIPT_DIR/qemuopts/$name.qemuopts")" + elif [ -f "$dir/$name.qemuopts" ]; then + test_qemuopts="$(cat "$dir/$name.qemuopts")" + fi + + local inline_qemuopts + inline_qemuopts="$(grep -E '^# *QEMUOPTS:' "$test_script" | sed 's/^# *QEMUOPTS://' | xargs || true)" + if [ -n "$inline_qemuopts" ]; then + test_qemuopts="$test_qemuopts $inline_qemuopts" + fi + + local test_timeout="$TIMEOUT" + local inline_timeout + inline_timeout="$(grep -E '^# *TIMEOUT:' "$test_script" | sed 's/^# *TIMEOUT://' | xargs || true)" + if [ -n "$inline_timeout" ]; then + test_timeout="$inline_timeout" + fi + + local is_reboot=0 + if grep -q -E '^# *REBOOT: *1' "$test_script" || [ -f "$dir/$name.reboot" ]; then + is_reboot=1 + fi + + local effective_timeout="$test_timeout" + if [ "$is_reboot" -eq 1 ]; then + effective_timeout="$(double_timeout "$test_timeout")" + fi + + local workdir + workdir="$(mktemp -d)" + if [ -z "$workdir" ] || [ ! -d "$workdir" ]; then + ktap_test_fail "$name (failed to create temporary directory)" + return 0 + fi + trap 'rm -rf "$workdir"' EXIT + + local test_applets=("${BUSYBOX_APPLETS[@]}") + local inline_applets + inline_applets="$(grep -E '^# *APPLETS:' "$test_script" | sed 's/^# *APPLETS://' | xargs || true)" + if [ -n "$inline_applets" ]; then + read -r -a extra_applets <<< "$inline_applets" || true + test_applets+=("${extra_applets[@]}") + fi + + local rootfs="$workdir/rootfs" + mkdir -p "$rootfs"/{bin,sbin,etc,proc,sys,dev,tmp} + + cp -L "$BUSYBOX" "$rootfs/bin/busybox" + chmod +x "$rootfs/bin/busybox" + (cd "$rootfs/bin" && for applet in "${test_applets[@]}"; do ln -sf busybox "$applet"; done) + + if command -v ldd >/dev/null 2>&1; then + for lib in $(ldd "$BUSYBOX" 2>/dev/null | grep -o '/[^ ]*' || true); do + if [ -f "$lib" ]; then + mkdir -p "$rootfs$(dirname "$lib")" + cp -L "$lib" "$rootfs$lib" 2>/dev/null || true + fi + done + fi + + cat << 'EOF' > "$rootfs/init" +#!/bin/sh +mount -t proc proc /proc 2>/dev/null +mount -t sysfs sys /sys 2>/dev/null +mount -t tracefs nodev /sys/kernel/tracing 2>/dev/null + +/bin/check_test.sh +RET=$? + +if [ $RET -eq 0 ]; then + echo "TEST RESULT: PASS" +else + echo "TEST RESULT: FAIL" +fi +sync +echo o > /proc/sysrq-trigger 2>/dev/null || true +poweroff -f 2>/dev/null || reboot -f 2>/dev/null || true +while true; do sleep 100 2>/dev/null || break; done +EOF + chmod +x "$rootfs/init" + + cp "$test_script" "$rootfs/bin/check_test.sh" + chmod +x "$rootfs/bin/check_test.sh" + + local initramfs="$workdir/initramfs.cpio" + (cd "$rootfs" && find . | cpio -o -H newc --quiet) > "$initramfs" + + if [ -n "$bconf_file" ] && [ -f "$bconf_file" ]; then + "$BOOTCONFIG" -a "$bconf_file" "$initramfs" > /dev/null + fi + + + local logfile + local base_cmdline="bootconfig console=tty0 console=ttyS0 panic=-1" + if [ -n "$test_cmdline" ]; then + base_cmdline="$base_cmdline $test_cmdline" + fi + + if [ -n "$LOGDIR" ]; then + mkdir -p "$LOGDIR" + logfile="$LOGDIR/$name.log" + base_cmdline="$base_cmdline dump_bconf" + else + logfile="$workdir/qemu.log" + base_cmdline="quiet $base_cmdline" + fi + + local qemu_args=() + if [ -n "$test_qemuopts" ]; then + read -r -d '' -a qemu_args <<< "$test_qemuopts" || true + fi + + if [ "$is_reboot" -ne 1 ]; then + qemu_args+=("-no-reboot") + fi + + timeout "$effective_timeout" "$QEMU" -kernel "$KERNEL" \ + -initrd "$initramfs" \ + -append "$base_cmdline" \ + -display none \ + -serial stdio \ + "${qemu_args[@]}" < /dev/null > "$logfile" 2>&1 || true + + if grep -q "TEST RESULT: PASS" "$logfile"; then + ktap_test_pass "$name" + else + ktap_test_fail "$name" + ktap_print_msg "QEMU Console Output for $name:" + while IFS= read -r line; do + ktap_print_msg "$line" + done < "$logfile" + fi + + rm -rf "$workdir" + trap - EXIT + return 0 +} + +TEST_SCRIPTS=() +if [ -d "$SCRIPT_DIR/tests" ]; then + while IFS= read -r f; do + [ -n "$f" ] && TEST_SCRIPTS+=("$f") + done < <(find "$SCRIPT_DIR/tests" -type f -name "*.sh" | sort) +fi + +TOTAL_TESTS=0 +for test_script in "${TEST_SCRIPTS[@]}"; do + name="$(basename "$test_script" .sh)" + if [ "$TARGET_TEST" != "all" ] && [ "$TARGET_TEST" != "$name" ]; then + continue + fi + TOTAL_TESTS=$((TOTAL_TESTS + 1)) +done + +ktap_print_header +ktap_set_plan "$TOTAL_TESTS" + +for test_script in "${TEST_SCRIPTS[@]}"; do + name="$(basename "$test_script" .sh)" + + if [ "$TARGET_TEST" != "all" ] && [ "$TARGET_TEST" != "$name" ]; then + continue + fi + + run_single_test "$test_script" +done + +ktap_finished
diff --git a/tools/testing/selftests/ftrace/boottime/tests/01-kprobe.sh b/tools/testing/selftests/ftrace/boottime/tests/01-kprobe.sh new file mode 100644 index 0000000..6ed7cac --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/01-kprobe.sh
@@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check kprobe bootconfig settings on tracefs +TRACEDIR="/sys/kernel/tracing" + +if [ -f /proc/bootconfig ] && grep -q "dump_bconf" /proc/cmdline 2>/dev/null; then + echo "=== /proc/bootconfig ===" + cat /proc/bootconfig + echo "========================" +fi + +if [ ! -d "$TRACEDIR/events/kprobes/vfs_read" ]; then + echo "FAIL: kprobe event kprobes/vfs_read does not exist" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/events/kprobes/vfs_read/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: kprobe event kprobes/vfs_read is not enabled ($ENABLE)" + exit 1 +fi + +echo "PASS: 01-kprobe" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/02-synth.sh b/tools/testing/selftests/ftrace/boottime/tests/02-synth.sh new file mode 100644 index 0000000..745c328 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/02-synth.sh
@@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check synthetic event bootconfig settings on tracefs +TRACEDIR="/sys/kernel/tracing" + +if [ -f /proc/bootconfig ] && grep -q "dump_bconf" /proc/cmdline 2>/dev/null; then + echo "=== /proc/bootconfig ===" + cat /proc/bootconfig + echo "========================" +fi + +if [ ! -d "$TRACEDIR/events/synthetic/boot_lat" ]; then + echo "FAIL: synthetic event synthetic/boot_lat does not exist" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/events/synthetic/boot_lat/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: synthetic event synthetic/boot_lat is not enabled ($ENABLE)" + exit 1 +fi + +echo "PASS: 02-synth" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/03-eprobe.sh b/tools/testing/selftests/ftrace/boottime/tests/03-eprobe.sh new file mode 100644 index 0000000..9b3b7ab --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/03-eprobe.sh
@@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check eprobe event bootconfig settings on tracefs +TRACEDIR="/sys/kernel/tracing" + +if [ -f /proc/bootconfig ] && grep -q "dump_bconf" /proc/cmdline 2>/dev/null; then + echo "=== /proc/bootconfig ===" + cat /proc/bootconfig + echo "========================" +fi + +if [ ! -d "$TRACEDIR/events/eprobes/ep_read" ]; then + echo "FAIL: eprobe event eprobes/ep_read does not exist" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/events/eprobes/ep_read/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: eprobe event eprobes/ep_read is not enabled ($ENABLE)" + exit 1 +fi + +echo "PASS: 03-eprobe" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/04-fprobe.sh b/tools/testing/selftests/ftrace/boottime/tests/04-fprobe.sh new file mode 100644 index 0000000..27e090c --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/04-fprobe.sh
@@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check fprobe event bootconfig settings on tracefs +TRACEDIR="/sys/kernel/tracing" + +if [ -f /proc/bootconfig ] && grep -q "dump_bconf" /proc/cmdline 2>/dev/null; then + echo "=== /proc/bootconfig ===" + cat /proc/bootconfig + echo "========================" +fi + +if [ ! -d "$TRACEDIR/events/fprobes/fp_read" ]; then + echo "FAIL: fprobe event fprobes/fp_read does not exist" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/events/fprobes/fp_read/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: fprobe event fprobes/fp_read is not enabled ($ENABLE)" + exit 1 +fi + +echo "PASS: 04-fprobe" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/05-tprobe.sh b/tools/testing/selftests/ftrace/boottime/tests/05-tprobe.sh new file mode 100644 index 0000000..810d340 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/05-tprobe.sh
@@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check tracepoint probe event bootconfig settings on tracefs +TRACEDIR="/sys/kernel/tracing" + +if [ -f /proc/bootconfig ] && grep -q "dump_bconf" /proc/cmdline 2>/dev/null; then + echo "=== /proc/bootconfig ===" + cat /proc/bootconfig + echo "========================" +fi + +if [ ! -d "$TRACEDIR/events/tracepoints/tp_sched" ]; then + echo "FAIL: tracepoint probe event tracepoints/tp_sched does not exist" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/events/tracepoints/tp_sched/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: tracepoint probe event tracepoints/tp_sched is not enabled ($ENABLE)" + exit 1 +fi + +echo "PASS: 05-tprobe" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/06-instance.sh b/tools/testing/selftests/ftrace/boottime/tests/06-instance.sh new file mode 100644 index 0000000..f6e8519 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/06-instance.sh
@@ -0,0 +1,30 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check instance bootconfig settings on tracefs +TRACEDIR="/sys/kernel/tracing" + +if [ -f /proc/bootconfig ] && grep -q "dump_bconf" /proc/cmdline 2>/dev/null; then + echo "=== /proc/bootconfig ===" + cat /proc/bootconfig + echo "========================" +fi + +if [ ! -d "$TRACEDIR/instances/foo" ]; then + echo "FAIL: trace instance foo does not exist" + exit 1 +fi + +if [ ! -d "$TRACEDIR/instances/foo/events/sched/sched_switch" ]; then + echo "FAIL: event sched_switch does not exist in instance foo" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/instances/foo/events/sched/sched_switch/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: event sched_switch is not enabled in instance foo ($ENABLE)" + exit 1 +fi + +echo "PASS: 06-instance" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/cmdline-01-ftrace.sh b/tools/testing/selftests/ftrace/boottime/tests/cmdline-01-ftrace.sh new file mode 100644 index 0000000..23a0b15 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/cmdline-01-ftrace.sh
@@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check ftrace= kernel command-line tracer setting +TRACEDIR="/sys/kernel/tracing" + +if [ ! -f "$TRACEDIR/current_tracer" ]; then + echo "FAIL: current_tracer does not exist" + exit 1 +fi + +read -r TRACER _ < "$TRACEDIR/current_tracer" +if [ "$TRACER" != "function" ]; then + echo "FAIL: current_tracer is '$TRACER', expected 'function'" + exit 1 +fi + +echo "PASS: cmdline-01-ftrace" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/cmdline-02-trace-event.sh b/tools/testing/selftests/ftrace/boottime/tests/cmdline-02-trace-event.sh new file mode 100644 index 0000000..b793907 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/cmdline-02-trace-event.sh
@@ -0,0 +1,26 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check trace_event= kernel command-line setting +TRACEDIR="/sys/kernel/tracing" + +if [ ! -d "$TRACEDIR/events/sched/sched_switch" ]; then + echo "FAIL: event sched:sched_switch does not exist" + exit 1 +fi + +if [ ! -d "$TRACEDIR/events/kmem/kmalloc" ]; then + echo "FAIL: event kmem:kmalloc does not exist" + exit 1 +fi + +ENABLE1=$(cat "$TRACEDIR/events/sched/sched_switch/enable") +ENABLE2=$(cat "$TRACEDIR/events/kmem/kmalloc/enable") + +if [ "$ENABLE1" != "1" ] || [ "$ENABLE2" != "1" ]; then + echo "FAIL: events not enabled (sched_switch=$ENABLE1, kmalloc=$ENABLE2)" + exit 1 +fi + +echo "PASS: cmdline-02-trace-event" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/cmdline-03-trace-buf-size.sh b/tools/testing/selftests/ftrace/boottime/tests/cmdline-03-trace-buf-size.sh new file mode 100644 index 0000000..4b936f8 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/cmdline-03-trace-buf-size.sh
@@ -0,0 +1,29 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# APPLETS: awk +# Check trace_buf_size= kernel command-line setting +TRACEDIR="/sys/kernel/tracing" + +if [ ! -f "$TRACEDIR/buffer_size_kb" ]; then + echo "FAIL: buffer_size_kb does not exist" + exit 1 +fi + +BUF_RAW=$(cat "$TRACEDIR/buffer_size_kb") +case "$BUF_RAW" in + *"expanded:"*) + BUFSIZE=$(echo "$BUF_RAW" | sed -n 's/.*expanded: *\([0-9]*\).*/\1/p') + ;; + *) + BUFSIZE=$(echo "$BUF_RAW" | awk '{print $1}') + ;; +esac + +if [ -z "$BUFSIZE" ] || [ "$BUFSIZE" -lt 2048 ]; then + echo "FAIL: buffer_size_kb is '$BUF_RAW', expected >= 2048" + exit 1 +fi + +echo "PASS: cmdline-03-trace-buf-size" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/cmdline-04-trace-options.sh b/tools/testing/selftests/ftrace/boottime/tests/cmdline-04-trace-options.sh new file mode 100644 index 0000000..53a65b5 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/cmdline-04-trace-options.sh
@@ -0,0 +1,23 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check trace_options= kernel command-line setting +TRACEDIR="/sys/kernel/tracing" + +if [ ! -f "$TRACEDIR/trace_options" ]; then + echo "FAIL: trace_options file does not exist" + exit 1 +fi + +if ! grep -qw "sym-addr" "$TRACEDIR/trace_options"; then + echo "FAIL: sym-addr option is not set in trace_options" + exit 1 +fi + +if ! grep -qw "verbose" "$TRACEDIR/trace_options"; then + echo "FAIL: verbose option is not set in trace_options" + exit 1 +fi + +echo "PASS: cmdline-04-trace-options" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/cmdline-05-trace-clock.sh b/tools/testing/selftests/ftrace/boottime/tests/cmdline-05-trace-clock.sh new file mode 100644 index 0000000..d6de9f6 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/cmdline-05-trace-clock.sh
@@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check trace_clock= kernel command-line setting +TRACEDIR="/sys/kernel/tracing" + +if [ ! -f "$TRACEDIR/trace_clock" ]; then + echo "FAIL: trace_clock file does not exist" + exit 1 +fi + +if ! grep -q '\[global\]' "$TRACEDIR/trace_clock"; then + CLOCK=$(cat "$TRACEDIR/trace_clock") + echo "FAIL: trace_clock is not set to global ($CLOCK)" + exit 1 +fi + +echo "PASS: cmdline-05-trace-clock" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/cmdline-06-trace-instance.sh b/tools/testing/selftests/ftrace/boottime/tests/cmdline-06-trace-instance.sh new file mode 100644 index 0000000..aae7f0a --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/cmdline-06-trace-instance.sh
@@ -0,0 +1,24 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check trace_instance= kernel command-line setting +TRACEDIR="/sys/kernel/tracing" + +if [ ! -d "$TRACEDIR/instances/bar" ]; then + echo "FAIL: trace instance bar does not exist" + exit 1 +fi + +if [ ! -d "$TRACEDIR/instances/bar/events/sched/sched_switch" ]; then + echo "FAIL: event sched_switch does not exist in instance bar" + exit 1 +fi + +ENABLE=$(cat "$TRACEDIR/instances/bar/events/sched/sched_switch/enable") +if [ "$ENABLE" != "1" ]; then + echo "FAIL: event sched_switch is not enabled in instance bar ($ENABLE)" + exit 1 +fi + +echo "PASS: cmdline-06-trace-instance" +exit 0
diff --git a/tools/testing/selftests/ftrace/boottime/tests/persistent-01-reserve-mem.sh b/tools/testing/selftests/ftrace/boottime/tests/persistent-01-reserve-mem.sh new file mode 100644 index 0000000..49abaf5 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/persistent-01-reserve-mem.sh
@@ -0,0 +1,28 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check persistent ring buffer across guest crash/reboot +# REBOOT: 1 +TRACEDIR="/sys/kernel/tracing" + +if [ ! -f "$TRACEDIR/instances/boot_map/trace" ]; then + echo "FAIL: persistent trace instance boot_map/trace file missing" + exit 1 +fi + +# Check if Boot 1 marker was already written +if grep -q "BOOT1_MARKER" "$TRACEDIR/instances/boot_map/trace" 2>/dev/null; then + # Second boot: verify persistent ring buffer content from first boot + echo "PASS: persistent-01-reserve-mem" + exit 0 +fi + +# First boot: write marker to persistent buffer and trigger kernel crash/reboot +echo "BOOT1_MARKER" > "$TRACEDIR/instances/boot_map/trace_marker" +sync + +# Trigger reboot to restart into second boot +echo b > /proc/sysrq-trigger 2>/dev/null || echo c > /proc/sysrq-trigger 2>/dev/null || true +sleep 5 +echo "FAIL: reboot trigger failed on first boot" +exit 1
diff --git a/tools/testing/selftests/ftrace/boottime/tests/persistent-02-backup-instance.sh b/tools/testing/selftests/ftrace/boottime/tests/persistent-02-backup-instance.sh new file mode 100644 index 0000000..0b2a2f0 --- /dev/null +++ b/tools/testing/selftests/ftrace/boottime/tests/persistent-02-backup-instance.sh
@@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2026, Google LLC. +# Check persistent backup instance across guest crash/reboot +# REBOOT: 1 +TRACEDIR="/sys/kernel/tracing" + +# Ensure both boot_map/trace and backup/trace files exist +if [ ! -f "$TRACEDIR/instances/boot_map/trace" ]; then + echo "FAIL: boot_map/trace file missing" + exit 1 +fi + +if [ ! -f "$TRACEDIR/instances/backup/trace" ]; then + echo "FAIL: backup/trace file missing" + exit 1 +fi + +# Check if BOOT1_MARKER is in boot_map/trace (indicates second boot) +if grep -q "BOOT1_MARKER" "$TRACEDIR/instances/boot_map/trace" 2>/dev/null; then + # Second boot: verify BOOT1_MARKER was copied into backup/trace from Boot 1 + if grep -q "BOOT1_MARKER" "$TRACEDIR/instances/backup/trace" 2>/dev/null; then + echo "PASS: persistent-02-backup-instance" + exit 0 + else + echo "FAIL: BOOT1_MARKER found in boot_map/trace" \ + "but missing from backup/trace on second boot" + exit 1 + fi +fi + +# First boot: write BOOT1_MARKER to boot_map and reboot via sysrq-trigger +echo "BOOT1_MARKER" > "$TRACEDIR/instances/boot_map/trace_marker" +sync + +# Trigger reboot to restart into second boot +echo b > /proc/sysrq-trigger 2>/dev/null || echo c > /proc/sysrq-trigger 2>/dev/null || true +sleep 5 +echo "FAIL: reboot trigger failed on first boot" +exit 1
diff --git a/tools/testing/selftests/ftrace/config b/tools/testing/selftests/ftrace/config index 544de0d..29f05f7 100644 --- a/tools/testing/selftests/ftrace/config +++ b/tools/testing/selftests/ftrace/config
@@ -1,3 +1,5 @@ +CONFIG_BOOT_CONFIG=y +CONFIG_BOOTTIME_TRACING=y CONFIG_BPF_SYSCALL=y CONFIG_DEBUG_INFO_BTF=y CONFIG_DEBUG_INFO_DWARF4=y @@ -8,22 +10,26 @@ CONFIG_FTRACE_SYSCALLS=y CONFIG_FUNCTION_GRAPH_RETVAL=y CONFIG_FUNCTION_PROFILER=y +CONFIG_FUNCTION_TRACER=y CONFIG_HIST_TRIGGERS=y CONFIG_IRQSOFF_TRACER=y CONFIG_KALLSYMS_ALL=y CONFIG_KPROBES=y CONFIG_KPROBE_EVENTS=y +CONFIG_MAGIC_SYSRQ=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_PREEMPTIRQ_DELAY_TEST=m CONFIG_PREEMPT_TRACER=y CONFIG_PROBE_EVENTS_BTF_ARGS=y +CONFIG_RESERVE_MEM=y CONFIG_SAMPLES=y CONFIG_SAMPLE_FTRACE_DIRECT=m CONFIG_SAMPLE_TRACE_EVENTS=m CONFIG_SAMPLE_TRACE_PRINTK=m CONFIG_SCHED_TRACER=y CONFIG_STACK_TRACER=y +CONFIG_SYNTH_EVENTS=y CONFIG_TRACER_SNAPSHOT=y CONFIG_UPROBES=y CONFIG_UPROBE_EVENTS=y
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc index 07b1177..03eb9e7 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc
@@ -3,7 +3,7 @@ # description: Test failure of registering kprobe on non unique symbol # requires: kprobe_events -SYMBOL='name_show' +SYMBOL='init_once' # We skip this test on kernel where SYMBOL is unique or does not exist. if [ "$(grep -c -E "[[:alnum:]]+ t ${SYMBOL}$" /proc/kallsyms)" -le '1' ]; then
diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile index 2f423de..d1b65fd 100644 --- a/tools/testing/selftests/hid/Makefile +++ b/tools/testing/selftests/hid/Makefile
@@ -148,6 +148,8 @@ $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \ $(TOOLSDIR)/lib/rbtree.c \ $(TOOLSDIR)/lib/zalloc.c \ $(TOOLSDIR)/lib/string.c \
diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index f2b2230..03d91f0 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile
@@ -3,7 +3,7 @@ include $(top_srcdir)/scripts/subarch.include ARCH ?= $(SUBARCH) -ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch)) +ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch powerpc)) # Top-level selftests allows ARCH=x86_64 :-( ifeq ($(ARCH),x86_64) override ARCH := x86
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index f43549a..3cb35a6 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -6,7 +6,6 @@ LIBKVM += lib/assert.c LIBKVM += lib/elf.c LIBKVM += lib/guest_modes.c -LIBKVM += lib/io.c LIBKVM += lib/kvm_util.c LIBKVM += lib/lru_gen_util.c LIBKVM += lib/memstress.c @@ -53,6 +52,11 @@ LIBKVM_loongarch += lib/loongarch/ucall.c LIBKVM_loongarch += lib/loongarch/exception.S +LIBKVM_powerpc += lib/powerpc/handlers.S +LIBKVM_powerpc += lib/powerpc/processor.c +LIBKVM_powerpc += lib/powerpc/ucall.c +LIBKVM_powerpc += lib/powerpc/hcall.c + # Non-compiled test targets TEST_PROGS_x86 += x86/nx_huge_pages_test.sh @@ -118,7 +122,7 @@ TEST_GEN_PROGS_x86 += x86/vmx_preemption_timer_test TEST_GEN_PROGS_x86 += x86/svm_vmcall_test TEST_GEN_PROGS_x86 += x86/svm_int_ctl_test -TEST_GEN_PROGS_x86 += x86/svm_nested_clear_efer_svme +TEST_GEN_PROGS_x86 += x86/svm_nested_efer_test TEST_GEN_PROGS_x86 += x86/svm_nested_shutdown_test TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa @@ -156,6 +160,7 @@ TEST_GEN_PROGS_x86 += x86/triple_fault_event_test TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test TEST_GEN_PROGS_x86 += x86/aperfmperf_test +TEST_GEN_PROGS_x86 += x86/guest_memfd_conversions_test TEST_GEN_PROGS_x86 += access_tracking_perf_test TEST_GEN_PROGS_x86 += coalesced_io_test TEST_GEN_PROGS_x86 += dirty_log_perf_test @@ -179,6 +184,7 @@ TEST_GEN_PROGS_arm64 += arm64/hello_el2 TEST_GEN_PROGS_arm64 += arm64/host_sve TEST_GEN_PROGS_arm64 += arm64/hypercalls +TEST_GEN_PROGS_arm64 += arm64/nv_pre_fault_memory_test TEST_GEN_PROGS_arm64 += arm64/external_aborts TEST_GEN_PROGS_arm64 += arm64/mmio_sign_ext TEST_GEN_PROGS_arm64 += arm64/page_fault_test @@ -207,6 +213,7 @@ TEST_GEN_PROGS_arm64 += mmu_stress_test TEST_GEN_PROGS_arm64 += rseq_test TEST_GEN_PROGS_arm64 += steal_time +TEST_GEN_PROGS_arm64 += pre_fault_memory_test TEST_GEN_PROGS_s390 = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_s390 += s390/memop @@ -224,6 +231,7 @@ TEST_GEN_PROGS_s390 += s390/irq_routing TEST_GEN_PROGS_s390 += mmu_stress_test TEST_GEN_PROGS_s390 += pre_fault_memory_test +TEST_GEN_PROGS_s390 += s390/irq_injection TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test @@ -254,6 +262,10 @@ TEST_GEN_PROGS_loongarch += set_memory_region_test TEST_GEN_PROGS_loongarch += steal_time +TEST_GEN_PROGS_powerpc = $(TEST_GEN_PROGS_COMMON) +TEST_GEN_PROGS_powerpc += access_tracking_perf_test +TEST_GEN_PROGS_powerpc += hardware_disable_test + SPLIT_TESTS += arch_timer SPLIT_TESTS += get-reg-list
diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c index 4415c94..eb69390 100644 --- a/tools/testing/selftests/kvm/access_tracking_perf_test.c +++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c
@@ -472,6 +472,16 @@ static int access_tracking_unreliable(void) } #endif /* + * PowerPC: Nested KVM (pSeries LPAR) can hide accessed/idle, since KVM + * doesn't explicitely flush the TLB for access bit update. L0 keeps + * translations cached so the guest won't see the idle bit cleared. + */ + if (kvm_is_ppc_pseries_lpar()) { + puts("Skipping idle page count sanity check, because the test is run nested"); + return 1; + } + + /* * When NUMA balancing is enabled, guest memory will be unmapped to get * NUMA faults, dropping the Accessed bits. */
diff --git a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c index d9c9377..d46799f 100644 --- a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c +++ b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
@@ -787,6 +787,30 @@ static void test_timers_in_the_past(enum arch_timer timer) } } +/* + * A cval within long_wait_ms of the counter's wrap while the counter sits at + * wait_ms: a deadline further away than the counter can reach. Unmasked, so + * an early ISTATUS shows up as an IRQ. + */ +static void test_timers_past_counter_wrap(enum arch_timer timer) +{ + u64 cval = -msec_to_cycles(test_args.long_wait_ms); + int i; + + for (i = 0; i < ARRAY_SIZE(sleep_method); i++) { + sleep_method_t sm = sleep_method[i]; + + local_irq_disable(); + set_counter(timer, msec_to_cycles(test_args.wait_ms)); + set_cval_irq(timer, cval, CTL_ENABLE); + sm(timer, msecs_to_usecs(test_args.wait_ms) + TIMEOUT_NO_IRQ_US); + local_irq_enable(); + isb(); + assert_irqs_handled(0); + timer_set_ctl(timer, CTL_IMASK); + } +} + static void test_long_timer_delays(enum arch_timer timer) { s32 tval = (s32)msec_to_cycles(test_args.long_wait_ms); @@ -806,6 +830,7 @@ static void guest_run_iteration(enum arch_timer timer) test_basic_functionality(timer); test_timers_sanity_checks(timer); + test_timers_past_counter_wrap(timer); test_timers_above_tval_max(timer); test_timers_in_the_past(timer);
diff --git a/tools/testing/selftests/kvm/arm64/debug-exceptions.c b/tools/testing/selftests/kvm/arm64/debug-exceptions.c index 7dc5f0b..9da6dc5 100644 --- a/tools/testing/selftests/kvm/arm64/debug-exceptions.c +++ b/tools/testing/selftests/kvm/arm64/debug-exceptions.c
@@ -387,7 +387,7 @@ static void guest_code_ss(int test_cnt) * Enable Single Step execution. Note! This _must_ be a bare * ucall as the ucall() path uses atomic operations to manage * the ucall structures, and the built-in "atomics" are usually - * implemented via exclusive access instructions. The exlusive + * implemented via exclusive access instructions. The exclusive * monitor is cleared on ERET, and so taking debug exceptions * during a LDREX=>STREX sequence will prevent forward progress * and hang the guest/test.
diff --git a/tools/testing/selftests/kvm/arm64/external_aborts.c b/tools/testing/selftests/kvm/arm64/external_aborts.c index d8fe17a..1a858d0 100644 --- a/tools/testing/selftests/kvm/arm64/external_aborts.c +++ b/tools/testing/selftests/kvm/arm64/external_aborts.c
@@ -75,6 +75,17 @@ static void vcpu_inject_serror(struct kvm_vcpu *vcpu) vcpu_events_set(vcpu, &events); } +static void vcpu_assert_serror_pending(struct kvm_vcpu *vcpu, bool pending) +{ + struct kvm_vcpu_events events; + + vcpu_events_get(vcpu, &events); + TEST_ASSERT_EQ(events.exception.serror_pending, pending); + + if (pending && vcpu_has_ras(vcpu)) + TEST_ASSERT_EQ(events.exception.serror_esr, EXPECTED_SERROR_ISS); +} + static void __vcpu_run_expect(struct kvm_vcpu *vcpu, unsigned int cmd) { struct ucall uc; @@ -134,6 +145,7 @@ static void test_mmio_abort(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -162,6 +174,7 @@ static void test_mmio_nisv(void) TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN"); TEST_ASSERT_EQ(errno, ENOSYS); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -185,6 +198,7 @@ static void test_mmio_nisv_abort(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -212,6 +226,7 @@ static void test_serror_masked(void) vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, true); kvm_vm_free(vm); } @@ -247,6 +262,7 @@ static void test_serror(void) vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -288,6 +304,7 @@ static void test_s1ptw_abort(void) *ptep |= bad_pa; vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -313,6 +330,7 @@ static void test_serror_emulated(void) vcpu_run_expect_sync(vcpu); vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -339,7 +357,7 @@ static void test_mmio_ease(void) pfr1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1)); if (!SYS_FIELD_GET(ID_AA64PFR1_EL1, DF2, pfr1)) { pr_debug("Skipping %s\n", __func__); - return; + goto done; } /* @@ -356,6 +374,8 @@ static void test_mmio_ease(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); +done: kvm_vm_free(vm); } @@ -394,6 +414,7 @@ static void test_serror_amo(void) vcpu_run_expect_sync(vcpu); vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); }
diff --git a/tools/testing/selftests/kvm/arm64/hypercalls.c b/tools/testing/selftests/kvm/arm64/hypercalls.c index 5d96cdf..87eade2 100644 --- a/tools/testing/selftests/kvm/arm64/hypercalls.c +++ b/tools/testing/selftests/kvm/arm64/hypercalls.c
@@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only -/* hypercalls: Check the ARM64's psuedo-firmware bitmap register interface. +/* hypercalls: Check the ARM64's pseudo-firmware bitmap register interface. * * The test validates the basic hypercall functionalities that are exposed - * via the psuedo-firmware bitmap register. This includes the registers' + * via the pseudo-firmware bitmap register. This includes the registers' * read/write behavior before and after the VM has started, and if the * hypercalls are properly masked or unmasked to the guest when disabled or - * enabled from the KVM userspace, respectively. + * enabled from the KVM userspace, respectively. It also checks the two + * SMCCC_ARCH_FEATURES queries whose answer SMCCC pins to SUCCESS. */ #include <errno.h> #include <linux/arm-smccc.h> @@ -97,6 +98,12 @@ static const struct test_hvc_info false_hvc_info[] = { TEST_HVC_INFO(ARM_SMCCC_HV_PV_TIME_FEATURES, ARM_SMCCC_TRNG_RND64), }; +/* SMCCC pins the answer to SUCCESS, whatever the bitmaps hold. */ +static const struct test_hvc_info always_hvc_info[] = { + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_VERSION_FUNC_ID), + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_ARCH_FEATURES_FUNC_ID), +}; + static void guest_test_hvc(const struct test_hvc_info *hc_info) { unsigned int i; @@ -128,6 +135,23 @@ static void guest_test_hvc(const struct test_hvc_info *hc_info) } } +static void guest_test_always_hvc(void) +{ + struct arm_smccc_res res; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(always_hvc_info); i++) { + memset(&res, 0, sizeof(res)); + do_smccc(always_hvc_info[i].func_id, always_hvc_info[i].arg1, + 0, 0, 0, 0, 0, 0, &res); + + __GUEST_ASSERT(res.a0 == SMCCC_RET_SUCCESS, + "a0 = 0x%lx, func_id = 0x%x, arg1 = 0x%lx, stage = %u", + res.a0, always_hvc_info[i].func_id, + always_hvc_info[i].arg1, stage); + } +} + static void guest_code(void) { while (stage != TEST_STAGE_END) { @@ -137,6 +161,7 @@ static void guest_code(void) case TEST_STAGE_HVC_IFACE_FEAT_DISABLED: case TEST_STAGE_HVC_IFACE_FEAT_ENABLED: guest_test_hvc(hvc_info); + guest_test_always_hvc(); break; case TEST_STAGE_HVC_IFACE_FALSE_INFO: guest_test_hvc(false_hvc_info); @@ -162,7 +187,7 @@ struct st_time { static void steal_time_init(struct kvm_vcpu *vcpu) { - u64 st_ipa = (ulong)ST_GPA_BASE; + u64 st_ipa = (unsigned long)ST_GPA_BASE; unsigned int gpages; gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE);
diff --git a/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c b/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c new file mode 100644 index 0000000..09c1db3 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c
@@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * nv_pre_fault_memory_test - Test KVM_PRE_FAULT_MEMORY on a vCPU whose + * last-run context is nested. + * + * The guest enters vEL2, sets up its EL2 translation configuration into the + * real EL1 registers then ERETs to vEL1 and exits to userspace so its vCPU + * last-run context is nested backed by a shadow stage 2 MMU. + * + * Assert that pre-faulting ignores that and targets the canonical stage-2 + * page tables only. + */ +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" +#include "ucall.h" + +#include <asm/sysreg.h> +#include <linux/sizes.h> + +#define TEST_MEM_SLOT 10 +#define TEST_MEM_SIZE SZ_2M +#define TEST_MEM_GPA SZ_1G + +static void guest_el1_code(void) +{ + u64 offset; + + GUEST_ASSERT_EQ(get_current_el(), 1); + + /* Exit to userspace with the vEL1 (nested) context live. */ + GUEST_SYNC(1); + + /* + * Touch the prefaulted range. vstage-2 is disabled, so the shadow + * stage-2 is a 1:1 view of the canonical IPA space. + */ + for (offset = 0; offset < TEST_MEM_SIZE; offset += SZ_4K) + READ_ONCE(*(u64 *)(TEST_MEM_GPA + offset)); + + GUEST_DONE(); +} + +static void guest_code(void) +{ + u64 sp; + + GUEST_ASSERT_EQ(get_current_el(), 2); + + /* + * Mirror the EL2 translation regime into the real EL1 registers so + * that vEL1 runs on the test's stage-1 page tables. With E2H=1, the + * _EL1 accessors read the EL2 registers, and the _EL12 accessors + * write the real EL1 registers. + */ + write_sysreg_s(read_sysreg(sctlr_el1), SYS_SCTLR_EL12); + write_sysreg_s(read_sysreg(tcr_el1), SYS_TCR_EL12); + write_sysreg_s(read_sysreg(ttbr0_el1), SYS_TTBR0_EL12); + write_sysreg_s(read_sysreg(mair_el1), SYS_MAIR_EL12); + write_sysreg_s(read_sysreg(cpacr_el1), SYS_CPACR_EL12); + + /* Run vEL1 on the same stack. */ + asm volatile("mov %0, sp" : "=r"(sp)); + write_sysreg(sp, sp_el1); + + /* + * Drop TGE so that vEL1 is a nested context rather than host EL0. + * KVM backs it with a shadow stage-2 MMU even though vstage-2 is + * disabled (HCR_EL2.VM=0). + */ + write_sysreg(read_sysreg(hcr_el2) & ~HCR_EL2_TGE, hcr_el2); + isb(); + + write_sysreg(PSR_MODE_EL1h | PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | + PSR_D_BIT, spsr_el2); + write_sysreg((u64)guest_el1_code, elr_el2); + asm volatile("eret"); + + GUEST_ASSERT(false); +} + +static void pre_fault(struct kvm_vcpu *vcpu, u64 gpa, u64 size) +{ + struct kvm_pre_fault_memory range = { + .gpa = gpa, + .size = size, + }; + int ret; + + do { + ret = __vcpu_ioctl(vcpu, KVM_PRE_FAULT_MEMORY, &range); + } while ((!ret && range.size) || + (ret < 0 && (errno == EINTR || errno == EAGAIN))); + + TEST_ASSERT(!ret, "KVM_PRE_FAULT_MEMORY failed, ret: %d errno: %d", + ret, errno); + TEST_ASSERT_EQ(range.size, 0); +} + +int main(void) +{ + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + u64 npages; + + TEST_REQUIRE(test_supports_el2()); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); + + vm = vm_create(1); + + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= BIT(KVM_ARM_VCPU_HAS_EL2); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + npages = TEST_MEM_SIZE / vm->page_size; + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, TEST_MEM_GPA, + TEST_MEM_SLOT, npages, 0); + virt_map(vm, TEST_MEM_GPA, TEST_MEM_GPA, npages); + + /* Run the guest until it has ERET'd from vEL2 to vEL1. */ + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_SYNC: + TEST_ASSERT_EQ(uc.args[1], 1); + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unhandled ucall: %ld", uc.cmd); + } + + /* + * The vCPU's last-run context is vEL1, so its hw_mmu is a shadow + * stage-2 MMU. + * + * Pre-faulting must ignore that and populate the canonical stage-2. + */ + pre_fault(vcpu, TEST_MEM_GPA, TEST_MEM_SIZE); + + /* Resume at vEL1 and touch the prefaulted range. */ + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unhandled ucall: %ld", uc.cmd); + } + + kvm_vm_free(vm); + return 0; +}
diff --git a/tools/testing/selftests/kvm/arm64/page_fault_test.c b/tools/testing/selftests/kvm/arm64/page_fault_test.c index 6bb3d82..b0e3fc8 100644 --- a/tools/testing/selftests/kvm/arm64/page_fault_test.c +++ b/tools/testing/selftests/kvm/arm64/page_fault_test.c
@@ -597,18 +597,16 @@ static void setup_memslots(struct kvm_vm *vm, struct test_params *p) vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, CODE_AND_DATA_MEMSLOT, code_npages, 0); - vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT; - vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT; + ____vm_override_mem_region(vm, MEM_REGION_CODE, CODE_AND_DATA_MEMSLOT); + ____vm_override_mem_region(vm, MEM_REGION_DATA, CODE_AND_DATA_MEMSLOT); - vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size, - PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, - p->test_desc->pt_memslot_flags); - vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT; + __vm_override_mem_region(vm, MEM_REGION_PT, p->src_type, data_gpa - pt_size, + PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, + p->test_desc->pt_memslot_flags); - vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT, - data_size / guest_page_size, - p->test_desc->data_memslot_flags); - vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; + __vm_override_mem_region(vm, MEM_REGION_TEST_DATA, p->src_type, data_gpa, + TEST_DATA_MEMSLOT, data_size / guest_page_size, + p->test_desc->data_memslot_flags); } static void setup_ucall(struct kvm_vm *vm)
diff --git a/tools/testing/selftests/kvm/arm64/sea_to_user.c b/tools/testing/selftests/kvm/arm64/sea_to_user.c index e96d898..d4af6e0 100644 --- a/tools/testing/selftests/kvm/arm64/sea_to_user.c +++ b/tools/testing/selftests/kvm/arm64/sea_to_user.c
@@ -81,25 +81,46 @@ static u64 translate_hva_to_hpa(unsigned long hva) static void write_einj_entry(const char *einj_path, u64 val) { - char cmd[256] = {0}; - FILE *cmdfile = NULL; + char buf[32]; + int fd, len, ret; - sprintf(cmd, "echo %#lx > %s", val, einj_path); - cmdfile = popen(cmd, "r"); + fd = open(einj_path, O_WRONLY); + if (fd < 0) + ksft_exit_fail_perror(einj_path); - if (pclose(cmdfile) == 0) - ksft_print_msg("echo %#lx > %s - done\n", val, einj_path); - else - ksft_exit_fail_perror("Failed to write EINJ entry"); + len = snprintf(buf, sizeof(buf), "%#lx\n", val); + ret = write(fd, buf, len); + if (ret != len) + ksft_exit_fail_perror(einj_path); + + close(fd); + ksft_print_msg("%#lx > %s - done\n", val, einj_path); } static void inject_uer(u64 hpa) { - if (access("/sys/firmware/acpi/tables/EINJ", R_OK) == -1) - ksft_test_result_skip("EINJ table no available in firmware"); + int fd; - if (access(EINJ_ETYPE, R_OK | W_OK) == -1) - ksft_test_result_skip("EINJ module probably not loaded?"); + /* + * EINJ is exposed only through debugfs, and opening it tells us why it + * is unusable far more reliably than access()-ing the ACPI EINJ table: + * reading that table needs CAP_SYS_ADMIN, so access() gives a false + * negative for unprivileged runs. Open the injection interface and let + * errno say whether the test cannot run (skip) or genuinely failed. + */ + fd = open(EINJ_ETYPE, O_WRONLY); + if (fd < 0) { + switch (errno) { + case ENOENT: + ksft_exit_skip("need CONFIG_ACPI_APEI_EINJ and firmware EINJ support\n"); + case EACCES: + case EPERM: + ksft_exit_skip("EINJ requires running as root\n"); + default: + ksft_exit_fail_perror(EINJ_ETYPE); + } + } + close(fd); write_einj_entry(EINJ_ETYPE, ERROR_TYPE_MEMORY_UER); write_einj_entry(EINJ_FLAGS, MASK_MEMORY_UER); @@ -194,13 +215,33 @@ static void run_vm(struct kvm_vm *vm, struct kvm_vcpu *vcpu) ksft_print_msg("Dump kvm_run info about KVM_EXIT_%s\n", exit_reason_str(run->exit_reason)); + + /* + * The guest's read of the injected location is expected to trap to KVM + * as an SEA. If it does not, the injected error was never placed as + * consumable poison: some firmware honours EINJ's notrigger request by + * arming the poison only as part of the (now skipped) trigger step, so + * nothing is left in memory for the guest to consume. The guest then + * reads back the sentinel value and reports it via GUEST_FAIL, which + * arm64 delivers as a ucall over MMIO (hence a KVM_EXIT_MMIO here). + * Treat that as "this platform cannot host the test" and skip, matching + * the requirement documented at the top of this file, rather than + * failing on a hardware/firmware limitation the test cannot control. + */ + if (run->exit_reason != KVM_EXIT_ARM_SEA && + get_ucall(vcpu, &uc) == UCALL_ABORT) { + ksft_print_msg("Guest consumed no SEA: %s", uc.buffer); + ksft_exit_skip("EINJ notrigger placed no consumable poison on this platform\n"); + } + + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA); + + /* arm_sea holds valid data only for a KVM_EXIT_ARM_SEA exit. */ ksft_print_msg("kvm_run.arm_sea: esr=%#llx, flags=%#llx\n", run->arm_sea.esr, run->arm_sea.flags); ksft_print_msg("kvm_run.arm_sea: gva=%#llx, gpa=%#llx\n", run->arm_sea.gva, run->arm_sea.gpa); - TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA); - esr = run->arm_sea.esr; TEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_LOW); TEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT); @@ -260,6 +301,12 @@ static struct kvm_vm *vm_create_with_sea_handler(struct kvm_vcpu **vcpu) alignment = max(backing_page_size, guest_page_size); num_guest_pages = VM_MEM_SIZE / guest_page_size; + /* + * The region is backed by 1GB hugepages; skip gracefully rather than + * failing with mmap() -ENOMEM if the host has none reserved. + */ + TEST_REQUIRE(get_free_hugepages(backing_page_size) >= VM_MEM_SIZE); + vm = __vm_create_with_one_vcpu(vcpu, num_guest_pages, guest_code); vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(*vcpu);
diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c index 7429a10..10849d2 100644 --- a/tools/testing/selftests/kvm/arm64/set_id_regs.c +++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c
@@ -13,6 +13,7 @@ #include "kvm_util.h" #include "processor.h" #include "test_util.h" +#include "vgic.h" #include <linux/bitfield.h> enum ftr_type { @@ -803,6 +804,107 @@ static void test_reset_preserves_id_regs(struct kvm_vcpu *vcpu) ksft_test_result_pass("%s\n", __func__); } +/* + * ID registers must stay immutable even when a vCPU's first KVM_RUN fails + * after finalization but before KVM_ARCH_FLAG_HAS_RAN_ONCE is set. + */ +static void test_idreg_frozen_after_failed_run(void) +{ + static const u32 imp_id_regs[] = { + SYS_MIDR_EL1, + SYS_REVIDR_EL1, + SYS_AIDR_EL1, + }; + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int r; + + if (!kvm_has_cap(KVM_CAP_ARM_PMU_V3)) { + ksft_print_msg("PMUv3 unsupported, cannot fail the first run\n"); + ksft_test_result_skip("%s\n", __func__); + return; + } + + /* Skip the default vGIC so the KVM_CREATE_DEVICE gate is reachable. */ + test_disable_default_vgic(); + + vm = vm_create(1); + vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + /* + * A PMUv3 vCPU left without PMU init is rejected by + * kvm_arm_pmu_v3_enable(), which runs after sysreg finalization. + */ + r = _vcpu_run(vcpu); + TEST_ASSERT(r < 0 && errno == EINVAL, + "first KVM_RUN should fail post-finalization: r=%d errno=%d", + r, errno); + + /* + * Feature ID registers: use values that would have been accepted before + * finalization, so that a rejection means the registers are final + * rather than the value being invalid. + */ + for (int i = 0; i < ARRAY_SIZE(test_regs); i++) { + const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits; + u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg); + u64 val = vcpu_get_reg(vcpu, reg); + + for (int j = 0; ftr_bits[j].type != FTR_END; j++) { + u64 ftr = (val & ftr_bits[j].mask) >> ftr_bits[j].shift; + u64 safe = get_safe_value(&ftr_bits[j], ftr); + u64 new_val; + + if (safe == ftr) + continue; + + new_val = (val & ~ftr_bits[j].mask) | + (safe << ftr_bits[j].shift); + + r = __vcpu_set_reg(vcpu, reg, new_val); + TEST_ASSERT(r < 0 && errno == EBUSY, + "%s write after failed first run: r=%d errno=%d", + ftr_bits[j].name, r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* A write matching the finalized value is still accepted. */ + vcpu_set_reg(vcpu, reg, val); + } + + /* + * The VM-wide implementation ID registers are gated separately. Bit 0 + * is within the writable mask of all three, so flipping it is a change + * KVM would otherwise accept. + */ + for (int i = 0; i < ARRAY_SIZE(imp_id_regs); i++) { + u64 reg = KVM_ARM64_SYS_REG(imp_id_regs[i]); + u64 val = vcpu_get_reg(vcpu, reg); + + r = __vcpu_set_reg(vcpu, reg, val ^ 1); + TEST_ASSERT(r < 0 && errno == EBUSY, + "implementation ID reg write after failed first run: r=%d errno=%d", + r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* Creating an in-kernel irqchip would change the ID registers too. */ + if (kvm_supports_vgic_v3()) { + r = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3); + TEST_ASSERT(r < 0 && errno == EBUSY, + "vGIC creation after failed first run: r=%d errno=%d", + r, errno); + } + + kvm_vm_free(vm); + ksft_test_result_pass("%s\n", __func__); +} + int main(void) { struct kvm_vcpu *vcpu; @@ -828,7 +930,7 @@ int main(void) ksft_print_header(); - test_cnt = 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST; + test_cnt = 4 + MPAM_IDREG_TEST + MTE_IDREG_TEST; for (i = 0; i < ARRAY_SIZE(test_regs); i++) for (j = 0; test_regs[i].ftr_bits[j].type != FTR_END; j++) test_cnt++; @@ -847,5 +949,7 @@ int main(void) kvm_vm_free(vm); + test_idreg_frozen_after_failed_run(); + ksft_finished(); }
diff --git a/tools/testing/selftests/kvm/arm64/vgic_init.c b/tools/testing/selftests/kvm/arm64/vgic_init.c index 47e34b4..5a30f3c 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_init.c +++ b/tools/testing/selftests/kvm/arm64/vgic_init.c
@@ -5,6 +5,7 @@ * Copyright (C) 2020, Red Hat, Inc. */ #include <linux/kernel.h> +#include <linux/sizes.h> #include <sys/syscall.h> #include <asm/kvm.h> #include <asm/kvm_para.h> @@ -13,12 +14,21 @@ #include "test_util.h" #include "kvm_util.h" +#include "gic.h" #include "processor.h" #include "vgic.h" #include "gic_v3.h" #define NR_VCPUS 4 +#define REDIST_RETRY_REGION0_BASE GICR_BASE_GPA +#define REDIST_RETRY_REGION1_BASE \ + (REDIST_RETRY_REGION0_BASE + 2 * KVM_VGIC_V3_REDIST_SIZE) +#define REDIST_RETRY_DIST_BASE \ + (REDIST_RETRY_REGION1_BASE + KVM_VGIC_V3_REDIST_SIZE) +#define REDIST_RETRY_REGION2_BASE \ + (REDIST_RETRY_DIST_BASE + KVM_VGIC_V3_DIST_SIZE) + #define REG_OFFSET(vcpu, offset) (((u64)vcpu << 32) | offset) #define VGIC_DEV_IS_V2(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V2) @@ -65,6 +75,23 @@ static void guest_code(void) GUEST_DONE(); } +static void guest_check_redist_retry(void) +{ + unsigned int i; + + /* The first three redistributors span adjacent regions 0 and 1. */ + for (i = 0; i < NR_VCPUS; i++) { + u64 base = i < 3 ? REDIST_RETRY_REGION0_BASE + + i * KVM_VGIC_V3_REDIST_SIZE : + REDIST_RETRY_REGION2_BASE; + u64 typer = readq((void *)(unsigned long)(base + GICR_TYPER)); + + GUEST_ASSERT_EQ(GICR_TYPER_CPU_NUMBER(typer), i); + } + + GUEST_DONE(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -73,6 +100,7 @@ static int run_vcpu(struct kvm_vcpu *vcpu) static struct vm_gic vm_gic_create_with_vcpus(u32 gic_dev_type, u32 nr_vcpus, + void *guest_code, struct kvm_vcpu *vcpus[]) { struct vm_gic v; @@ -338,7 +366,7 @@ static void test_vgic_then_vcpus(u32 gic_dev_type) struct vm_gic v; int ret, i; - v = vm_gic_create_with_vcpus(gic_dev_type, 1, vcpus); + v = vm_gic_create_with_vcpus(gic_dev_type, 1, guest_code, vcpus); subtest_dist_rdist(&v); @@ -359,7 +387,8 @@ static void test_vcpus_then_vgic(u32 gic_dev_type) struct vm_gic v; int ret; - v = vm_gic_create_with_vcpus(gic_dev_type, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(gic_dev_type, NR_VCPUS, guest_code, + vcpus); subtest_dist_rdist(&v); @@ -411,7 +440,8 @@ static void test_v3_new_redist_regions(void) u64 addr; int ret; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); @@ -422,7 +452,8 @@ static void test_v3_new_redist_regions(void) /* step2 */ - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); addr = REDIST_REGION_ATTR_ADDR(1, 0x280000, 0, 2); @@ -436,7 +467,8 @@ static void test_v3_new_redist_regions(void) /* step 3 */ - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, @@ -457,6 +489,70 @@ static void test_v3_new_redist_regions(void) vm_gic_destroy(&v); } +static void test_v3_redist_region_retry(void) +{ + struct kvm_vcpu *vcpus[NR_VCPUS]; + struct vm_gic v; + struct ucall uc; + u64 addr; + int ret; + + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_check_redist_retry, vcpus); + + addr = REDIST_REGION_ATTR_ADDR(2, REDIST_RETRY_REGION0_BASE, 0, 0); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_REGION1_BASE, 0, 1); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + addr = REDIST_RETRY_DIST_BASE; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_DIST, &addr); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_DIST_BASE, 0, 2); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, + &addr); + TEST_ASSERT(ret && errno == EINVAL, + "register redist region colliding with dist"); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_REGION2_BASE, 0, 2); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + virt_map(v.vm, REDIST_RETRY_REGION0_BASE, REDIST_RETRY_REGION0_BASE, + vm_calc_num_guest_pages(v.vm->mode, + 3 * KVM_VGIC_V3_REDIST_SIZE)); + virt_map(v.vm, REDIST_RETRY_REGION2_BASE, REDIST_RETRY_REGION2_BASE, + vm_calc_num_guest_pages(v.vm->mode, + KVM_VGIC_V3_REDIST_SIZE)); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + vcpu_run(vcpus[0]); + switch (get_ucall(vcpus[0], &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_NONE: + if (vcpus[0]->run->exit_reason == KVM_EXIT_MMIO) + TEST_FAIL("Unexpected MMIO exit at 0x%llx", + vcpus[0]->run->mmio.phys_addr); + fallthrough; + default: + TEST_FAIL("Unexpected ucall %lu, exit_reason %u", + uc.cmd, vcpus[0]->run->exit_reason); + } + + vm_gic_destroy(&v); +} + static void test_v3_typer_accesses(void) { struct vm_gic v; @@ -608,7 +704,8 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void) int ret, i; u64 addr; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, guest_code, + vcpus); /* Set space for 3 redists, we have 1 vcpu, so this succeeds. */ addr = max_phys_size - (3 * 2 * 0x10000); @@ -641,7 +738,8 @@ static void test_v3_its_region(void) u64 addr; int its_fd, ret; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); its_fd = kvm_create_device(v.vm, KVM_DEV_TYPE_ARM_VGIC_ITS); addr = 0x401000; @@ -684,7 +782,8 @@ static void test_v3_nassgicap(void) u32 typer2; int ret; - vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); kvm_device_attr_get(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, GICD_TYPER2, &typer2); has_nassgicap = typer2 & GICD_TYPER2_nASSGIcap; @@ -978,6 +1077,7 @@ void run_tests(u32 gic_dev_type) if (VGIC_DEV_IS_V3(gic_dev_type)) { test_v3_new_redist_regions(); + test_v3_redist_region_retry(); test_v3_typer_accesses(); test_v3_last_bit_redist_regions(); test_v3_last_bit_single_rdist();
diff --git a/tools/testing/selftests/kvm/arm64/vgic_its_save.c b/tools/testing/selftests/kvm/arm64/vgic_its_save.c index 864da01..fae8a6c 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_its_save.c +++ b/tools/testing/selftests/kvm/arm64/vgic_its_save.c
@@ -279,15 +279,15 @@ static void setup_memslot(size_t sz) size_t pages = sz / vm->page_size; gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, - TEST_MEMSLOT_INDEX, pages, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + gpa_base, TEST_MEMSLOT_INDEX, pages); } static gpa_t alloc_64k(size_t nr) { size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); - gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, + MEM_REGION_TEST_EXTRA); TEST_ASSERT(IS_ALIGNED(gpa, SZ_64K), "Allocation at 0x%lx is not 64K aligned, GITS_BASER cannot address it",
diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c index 9a69c43..45a87ba 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
@@ -23,7 +23,6 @@ #define GIC_LPI_OFFSET 8192 static size_t nr_iterations = 1000; -static gpa_t gpa_base; static struct kvm_vm *vm; static struct kvm_vcpu **vcpus; @@ -143,6 +142,7 @@ static void guest_code(size_t nr_lpis) static void setup_memslot(void) { + gpa_t gpa_base; size_t pages; size_t sz; @@ -164,8 +164,8 @@ static void setup_memslot(void) pages = sz / vm->page_size; gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, - TEST_MEMSLOT_INDEX, pages, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + gpa_base, TEST_MEMSLOT_INDEX, pages); } #define LPI_PROP_DEFAULT_PRIO 0xa0 @@ -191,28 +191,25 @@ static void setup_test_data(void) gpa_t cmdq_base; test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); test_data.collection_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); - cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, MEM_REGION_TEST_EXTRA); virt_map(vm, cmdq_base, cmdq_base, pages_per_64k); test_data.cmdq_base = cmdq_base; test_data.cmdq_base_va = (void *)cmdq_base; test_data.itt_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_devices, - gpa_base, TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); test_data.lpi_prop_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); configure_lpis(); test_data.lpi_pend_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_cpus, - gpa_base, TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); sync_global_to_guest(vm, test_data); }
diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 96cfd6b..d5df742 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c
@@ -13,6 +13,32 @@ #include "vgic.h" #define NR_VCPUS 1 +#define VGIC_V5_LPI_NR_VCPUS 2 +#define VGIC_V5_SPARSE_NR_VCPUS 2 +#define VGIC_V5_SPARSE_VCPU0_ID 17 +#define VGIC_V5_SPARSE_VCPU1_ID 3 +#define VGIC_V5_LPI_MEMSLOT 1 +#define VGIC_V5_NR_PRIVATE_IRQS 64 +#define VGIC_V5_DEFAULT_NR_SPIS 32 +#define VGIC_V5_MAX_NR_SPIS BIT(10) +#define VGIC_V5_IRS_SIZE 0x20000 +#define VGIC_V5_IRS_WAIT_RETRIES 1000000 +#define VGIC_V5_LPI_IST_BASE_GPA (GICV5_IRS_CONFIG_BASE_GPA + VGIC_V5_IRS_SIZE) +#define VGIC_V5_LPI_IST_SIZE 0x10000 +#define LPI_TEST_TO_VPE1 0 +#define LPI_TEST_TO_VPE0 1 +#define RESTORE_TEST_LPI_PENDING 0 +#define RESTORE_TEST_LPI_ENABLED 1 +#define RESTORE_TEST_SPI_PRIO_HIGH 0 +#define RESTORE_TEST_SPI_PRIO_LOW 1 +#define RESTORE_TEST_SPI_PENDING_DISABLED 2 +#define RESTORE_TEST_SPI_ENABLED 3 +#define VGIC_V5_IST_PRIO_HIGH 0x08 +#define VGIC_V5_IST_PRIO_LOW 0x10 + +static u64 max_phys_size; + +#define VGIC_V5_TEST_IST_BASER_GPA 0x10000000ULL struct vm_gic { struct kvm_vm *vm; @@ -24,8 +50,200 @@ struct vm_gic { #define GUEST_CMD_IRQ_DIEOI 11 #define GUEST_CMD_IS_AWAKE 12 #define GUEST_CMD_IS_READY 13 +#define GUEST_CMD_LPI_SENT 14 +#define GUEST_CMD_LPI_REPLIED 15 +#define GUEST_CMD_IST_SOURCE_READY 16 +#define GUEST_CMD_IST_SPI_INJECT_READY 17 +#define GUEST_CMD_IST_LPI_INJECT_READY 18 -static void guest_irq_handler(struct ex_regs *regs) +static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, + void *lpi_ist, size_t lpi_size) +{ + return (struct kvm_vgic_v5_ist) { + .spi_ist_addr = (uintptr_t)spi_ist, + .spi_ist_size = spi_size, + .lpi_ist_addr = (uintptr_t)lpi_ist, + .lpi_ist_size = lpi_size, + }; +} + +static u32 spi_line_expected; +static bool spi_line_level_sensitive; +static bool lpi_ist_ready; +static u32 ist_restore_irq_count[VGIC_V5_LPI_NR_VCPUS]; + +static u64 gicv5_hwirq(u32 type, u32 intid) +{ + return FIELD_PREP(GICV5_HWIRQ_TYPE, type) | + FIELD_PREP(GICV5_HWIRQ_ID, intid); +} + +static u64 gicv5_lpi_hwirq(u32 lpi) +{ + return gicv5_hwirq(GICV5_HWIRQ_TYPE_LPI, lpi); +} + +static u64 gicv5_spi_hwirq(u32 spi) +{ + return gicv5_hwirq(GICV5_HWIRQ_TYPE_SPI, spi); +} + +static void gicv5_setup_and_enable_hwirq(u64 hwirq, u32 target_vpe) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, + GICV5_IRQ_DEFAULT_PRI); + gic_insn(val, CDPRI); + + val = hwirq | FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, target_vpe); + gic_insn(val, CDAFF); + + gic_insn(hwirq, CDEN); +} + +static void gicv5_set_hwirq_priority(u64 hwirq, u32 priority) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority); + gic_insn(val, CDPRI); +} + +static void gicv5_set_hwirq_affinity(u64 hwirq, u32 target_vpe) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, target_vpe); + gic_insn(val, CDAFF); +} + +static void gicv5_enable_hwirq(u64 hwirq) +{ + gic_insn(hwirq, CDEN); +} + +static void gicv5_enable_spi(u32 spi, u32 target_vpe) +{ + gicv5_setup_and_enable_hwirq(gicv5_spi_hwirq(spi), target_vpe); +} + +static void gicv5_enable_lpi(u32 lpi, u32 target_vpe) +{ + gicv5_setup_and_enable_hwirq(gicv5_lpi_hwirq(lpi), target_vpe); +} + +static void gicv5_send_lpi(u32 lpi) +{ + u64 hwirq = gicv5_lpi_hwirq(lpi); + + gic_insn(hwirq | GICV5_GIC_CDPEND_PENDING_MASK, CDPEND); +} + +static void gicv5_wait_for_irs_idle(u32 reg, u32 idle) +{ + int i; + + for (i = 0; i < VGIC_V5_IRS_WAIT_RETRIES; i++) { + if (readl(GICV5_IRS_CONFIG_BASE_GVA + reg) & idle) + return; + + cpu_relax(); + } + + GUEST_FAIL("IRS operation did not become idle"); +} + +static void gicv5_configure_spi(u32 spi, bool level) +{ + u32 val; + + val = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, spi); + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_SPI_SELR); + gicv5_wait_for_irs_idle(GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE); + + val = level ? GICV5_IRS_SPI_CFGR_TM : 0; + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_SPI_CFGR); + gicv5_wait_for_irs_idle(GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE); +} + +static u32 gicv5_lpi_istsz(u32 idr2, u32 lpi_id_bits) +{ + if (!(idr2 & GICV5_IRS_IDR2_ISTMD)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (lpi_id_bits >= FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +static void gicv5_configure_lpi_ist(void) +{ + u32 idr2, min_lpi_id_bits, lpi_id_bits, id_bits, istsz; + u64 val; + + idr2 = readl(GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IDR2); + GUEST_ASSERT(idr2 & GICV5_IRS_IDR2_LPI); + + min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); + id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); + /* Default to 32 LPIs, unless the min requirement is higher */ + lpi_id_bits = max(5U, min_lpi_id_bits); + GUEST_ASSERT(lpi_id_bits <= id_bits); + + istsz = gicv5_lpi_istsz(idr2, lpi_id_bits); + /* + * We allocate a fixed-size IST buffer on the host to avoid dynamic + * allocation from the guest. Make sure it fits the linear IST described + * by IDR2 before programming IRS_IST_BASER. + */ + GUEST_ASSERT((BIT_ULL(lpi_id_bits) << (istsz + 2)) <= VGIC_V5_LPI_IST_SIZE); + + val = FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits) | + FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz); + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_CFGR); + + val = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + VGIC_V5_LPI_IST_BASE_GPA >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + val |= GICV5_IRS_IST_BASER_VALID; + writeq(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_BASER); + + GUEST_ASSERT_EQ(readl(GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_STATUSR), + GICV5_IRS_IST_STATUSR_IDLE); + + /* The address is immutable while the BASER is valid. */ + writeq(val ^ FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1), + GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_BASER); + GUEST_ASSERT_EQ(readq(GICV5_IRS_CONFIG_BASE_GVA + + GICV5_IRS_IST_BASER), val); +} + +static void gicv5_enable_irs(void) +{ + writel(GICV5_IRS_CR0_IRSEN, + GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_CR0); + gicv5_wait_for_irs_idle(GICV5_IRS_CR0, GICV5_IRS_CR0_IDLE); +} + +static void gicv5_configure_test_lpis(void) +{ + gicv5_enable_lpi(LPI_TEST_TO_VPE1, 1); + gicv5_enable_lpi(LPI_TEST_TO_VPE0, 0); +} + +static void gicv5_set_pcr(u32 priority) +{ + u64 pcr; + + pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, priority); + write_sysreg_s(pcr, SYS_ICC_PCR_EL1); + isb(); +} + +static void guest_ppi_irq_handler(struct ex_regs *regs) { bool valid; u32 hwirq; @@ -78,9 +296,284 @@ static void guest_code(void) GUEST_SYNC(GUEST_CMD_IS_READY); /* Loop forever waiting for interrupts */ - while (1); + for (;;) + cpu_relax(); } +static void guest_iaffid_code(void) +{ + GUEST_SYNC(FIELD_GET(ICC_IAFFIDR_EL1_IAFFID, + read_sysreg_s(SYS_ICC_IAFFIDR_EL1))); + GUEST_DONE(); +} + +static void guest_spi_irq_handler(struct ex_regs *regs) +{ + bool valid; + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + valid = GICV5_GICR_CDIA_VALID(ia); + + if (!valid) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + GUEST_ASSERT_EQ(hwirq, gicv5_spi_hwirq(READ_ONCE(spi_line_expected))); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + GUEST_DONE(); +} + +static void guest_spi_line_code(void) +{ + local_irq_disable(); + + gicv5_enable_irs(); + gicv5_cpu_enable_interrupts(); + gicv5_configure_spi(READ_ONCE(spi_line_expected), + READ_ONCE(spi_line_level_sensitive)); + gicv5_enable_spi(READ_ONCE(spi_line_expected), 0); + + local_irq_enable(); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + while (1) + wfi(); +} + +static void guest_lpi_irq_handler(struct ex_regs *regs) +{ + u32 vcpu_id = guest_get_vcpuid(); + u32 expected_lpi = vcpu_id ? LPI_TEST_TO_VPE1 : LPI_TEST_TO_VPE0; + u64 expected_hwirq = gicv5_lpi_hwirq(expected_lpi); + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + if (!GICV5_GICR_CDIA_VALID(ia)) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + GUEST_ASSERT_EQ(hwirq, expected_hwirq); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + if (vcpu_id) { + gicv5_send_lpi(LPI_TEST_TO_VPE0); + GUEST_SYNC(GUEST_CMD_LPI_REPLIED); + while (1) + wfi(); + } + + GUEST_DONE(); +} + +static void guest_unexpected_irq_handler(struct ex_regs *regs) +{ + GUEST_FAIL("Unexpected IRQ"); +} + +static void guest_lpi_code(void) +{ + u32 vcpu_id = guest_get_vcpuid(); + + local_irq_disable(); + + if (!vcpu_id) { + gicv5_enable_irs(); + gicv5_configure_lpi_ist(); + gicv5_configure_test_lpis(); + WRITE_ONCE(lpi_ist_ready, true); + } + + /* Go bang if we run VCPU1 before VCPU0 */ + GUEST_ASSERT(READ_ONCE(lpi_ist_ready)); + + gicv5_cpu_enable_interrupts(); + local_irq_enable(); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + if (!vcpu_id) { + gicv5_send_lpi(LPI_TEST_TO_VPE1); + GUEST_SYNC(GUEST_CMD_LPI_SENT); + } + + while (1) + wfi(); +} + +static void restore_test_setup_hwirq(u64 hwirq, u32 target_vpe, + u32 priority, bool enable) +{ + gicv5_set_hwirq_priority(hwirq, priority); + gicv5_set_hwirq_affinity(hwirq, target_vpe); + + if (enable) + gicv5_enable_hwirq(hwirq); +} + +/* + * The source VM configures SPI and LPI state for the IST save/restore test. + * It leaves interrupts disabled, allowing userspace to save the state before + * the restore VM enables interrupts and checks delivery. + */ +static void guest_ist_source_code(void) +{ + local_irq_disable(); + + gicv5_enable_irs(); + gicv5_configure_lpi_ist(); + + restore_test_setup_hwirq(gicv5_lpi_hwirq(RESTORE_TEST_LPI_PENDING), + 1, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_lpi_hwirq(RESTORE_TEST_LPI_ENABLED), + 1, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_HIGH), + 0, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_LOW), + 0, VGIC_V5_IST_PRIO_LOW, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_ENABLED), + 0, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED), + 0, VGIC_V5_IST_PRIO_HIGH, false); + + gicv5_send_lpi(RESTORE_TEST_LPI_PENDING); + + GUEST_SYNC(GUEST_CMD_IST_SOURCE_READY); + + while (1) + wfi(); +} + +/* + * This is the restore VM for the IST save/restore test. The IRS regs and ISTs + * have been restored before this code runs. Based on which VPE is running, a + * different set of interrupts is expected. We check that we get what we + * expected before terminating the test. + * + * The expected interrupts for VPE0 are: + * - High-priority SPI, pending and enabled before save. + * - Low-priority SPI, pending and enabled before save. This is only delivered + * after the guest lowers the masked priority via ICC_PCR_EL1. + * - SPI that was pending but disabled before save. This is only delivered + * after the guest enables it after restore. + * - SPI that was enabled but not pending before save. Userspace injects this + * after restore to check that the enable state and affinity were restored. + * + * The expected interrupts for VPE1 are: + * - LPI that was pending and enabled before save. + * - LPI that was enabled but not pending before save. The guest injects this + * after restore to check that the enable state and affinity were restored. + */ +static void guest_ist_restore_irq_handler(struct ex_regs *regs) +{ + u32 vcpu_id = guest_get_vcpuid(); + u32 count = ist_restore_irq_count[vcpu_id]; + u64 hwirq, expected_hwirq = 0; + u64 ia; + + ia = gicr_insn(CDIA); + if (!GICV5_GICR_CDIA_VALID(ia)) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + if (!vcpu_id) { + switch (count) { + case 0: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_HIGH); + break; + case 1: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_LOW); + break; + case 2: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED); + break; + case 3: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_ENABLED); + break; + default: + GUEST_FAIL("Unexpected VPE0 IST restore interrupt"); + } + } else { + switch (count) { + case 0: + expected_hwirq = gicv5_lpi_hwirq(RESTORE_TEST_LPI_PENDING); + break; + case 1: + expected_hwirq = gicv5_lpi_hwirq(RESTORE_TEST_LPI_ENABLED); + break; + default: + GUEST_FAIL("Unexpected VPE1 IST restore interrupt"); + } + } + + GUEST_ASSERT_EQ(hwirq, expected_hwirq); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + ist_restore_irq_count[vcpu_id]++; + + if (!vcpu_id) { + switch (count) { + case 0: + gicv5_set_pcr(VGIC_V5_IST_PRIO_LOW); + break; + case 1: + gicv5_enable_hwirq( + gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED)); + break; + case 2: + GUEST_SYNC(GUEST_CMD_IST_SPI_INJECT_READY); + break; + case 3: + GUEST_DONE(); + } + } else { + switch (count) { + case 0: + gicv5_send_lpi(RESTORE_TEST_LPI_ENABLED); + GUEST_SYNC(GUEST_CMD_IST_LPI_INJECT_READY); + break; + case 1: + GUEST_DONE(); + } + } +} + +static void guest_ist_restore_code(void) +{ + local_irq_disable(); + + gicv5_cpu_enable_interrupts(); + + if (!guest_get_vcpuid()) + gicv5_set_pcr(VGIC_V5_IST_PRIO_HIGH); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + local_irq_enable(); + + while (1) + wfi(); +} /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) @@ -94,12 +587,1005 @@ static void vm_gic_destroy(struct vm_gic *v) kvm_vm_free(v->vm); } +static void vgic_v5_map_irs(struct kvm_vm *vm) +{ + unsigned int nr_irs_pages; + + nr_irs_pages = vm_calc_num_guest_pages(vm->mode, VGIC_V5_IRS_SIZE); + + /* Map the IRS at VA == IPA so guest MMIO writes hit the IRS IODEV. */ + virt_map(vm, GICV5_IRS_CONFIG_BASE_GPA, + GICV5_IRS_CONFIG_BASE_GPA, nr_irs_pages); +} + +static u32 vgic_v5_irq_line_payload(u32 type, u32 num) +{ + return (type << KVM_ARM_IRQ_TYPE_SHIFT) | + FIELD_PREP(KVM_ARM_IRQ_NUM_MASK, num); +} + +static int __vgic_v5_irq_line(struct kvm_vm *vm, u32 type, u32 num, int level) +{ + return _kvm_irq_line(vm, vgic_v5_irq_line_payload(type, num), level); +} + +static void vgic_v5_spi_line(struct kvm_vm *vm, u32 spi, int level) +{ + int ret = __vgic_v5_irq_line(vm, KVM_ARM_IRQ_TYPE_SPI, spi, level); + + TEST_ASSERT(!ret, "KVM_IRQ_LINE failed for SPI %u level %d", + spi, level); +} + +static void vgic_v5_spi_line_vm_create(struct vm_gic *v, + struct kvm_vcpu **vcpu, + u32 nr_spis) +{ + u64 attr; + + v->gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v->vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); + *vcpu = vm_vcpu_add(v->vm, 0, guest_spi_line_code); + TEST_ASSERT(*vcpu, "Failed to create vCPU"); + + vm_init_descriptor_tables(v->vm); + vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, + guest_spi_irq_handler); + vcpu_init_descriptor_tables(*vcpu); + + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0, + &nr_spis); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v->vm); + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); +} + +static void vgic_v5_lpi_ist_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS], + void (*guest_code)(void), + void (*irq_handler)(struct ex_regs *)) +{ + unsigned int nr_lpi_ist_pages; + u64 attr; + int i; + + v->gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v->vm = __vm_create(VM_SHAPE_DEFAULT, VGIC_V5_LPI_NR_VCPUS, 0); + v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); + + for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) { + vcpus[i] = vm_vcpu_add(v->vm, i, guest_code); + TEST_ASSERT(vcpus[i], "Failed to create vCPU"); + } + + vm_init_descriptor_tables(v->vm); + vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, + irq_handler); + + for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) + vcpu_init_descriptor_tables(vcpus[i]); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v->vm); + + nr_lpi_ist_pages = vm_calc_num_guest_pages(v->vm->mode, + VGIC_V5_LPI_IST_SIZE); + vm_userspace_mem_region_add(v->vm, VM_MEM_SRC_ANONYMOUS, + VGIC_V5_LPI_IST_BASE_GPA, + VGIC_V5_LPI_MEMSLOT, + nr_lpi_ist_pages, 0); + /* + * Map the IST at VA == IPA so the guest can program the same BASER + * address. + */ + virt_map(v->vm, VGIC_V5_LPI_IST_BASE_GPA, + VGIC_V5_LPI_IST_BASE_GPA, nr_lpi_ist_pages); + + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); +} + +static void vgic_v5_lpi_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]) +{ + vgic_v5_lpi_ist_vm_create(v, vcpus, guest_lpi_code, + guest_lpi_irq_handler); +} + +static void vgic_v5_ist_restore_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS], + void (*guest_code)(void), + void (*irq_handler)(struct ex_regs *)) +{ + vgic_v5_lpi_ist_vm_create(v, vcpus, guest_code, irq_handler); +} + +static void vgic_v5_run_spi_line_test(u32 nr_spis, u32 expected_spi, + bool level_sensitive, + bool lower_before_run) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + struct ucall uc; + int ret; + + spi_line_expected = expected_spi; + spi_line_level_sensitive = level_sensitive; + + vgic_v5_spi_line_vm_create(&v, &vcpu, nr_spis); + + sync_global_to_guest(v.vm, spi_line_expected); + sync_global_to_guest(v.vm, spi_line_level_sensitive); + + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 IRQ_LINE VM"); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 IRQ_LINE guest did not become ready"); + + vgic_v5_spi_line(v.vm, expected_spi, 1); + /* + * For edge-triggered SPIs, a following low transition must be ignored: + * once the edge has made the SPI pending, it cannot be recalled. This + * allows us to test that. + */ + if (lower_before_run) + vgic_v5_spi_line(v.vm, expected_spi, 0); + + while (1) { + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 IRQ_LINE VM"); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_DONE: + goto done; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } + +done: + vm_gic_destroy(&v); +} + +static void vgic_v5_expect_sync(struct kvm_vcpu *vcpu, u64 cmd, + const char *msg) +{ + struct ucall uc; + + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == cmd, + "%s", msg); +} + +struct vgic_region_attr { + u64 attr; + u64 size; + u64 alignment; +}; + +static const struct vgic_region_attr gic_v5_irs_region = { + .attr = KVM_VGIC_V5_ADDR_TYPE_IRS, + .size = GICV5_IRS_SIZE, + .alignment = GICV5_IRS_ALIGN, +}; + +struct vgic_sysreg_attr { + const char *name; + u32 encoding; +}; + +struct vgic_irs_reg_attr { + const char *name; + u64 attr; +}; + +#define PACK_SR(r) \ + ((sys_reg_Op0(r) << 14) | \ + (sys_reg_Op1(r) << 11) | \ + (sys_reg_CRn(r) << 7) | \ + (sys_reg_CRm(r) << 3) | \ + (sys_reg_Op2(r))) + +#define SR(r) \ + { \ + .name = #r, \ + .encoding = r, \ + } + +#define IRS_REG(r) \ + { \ + .name = #r, \ + .attr = r, \ + } + +static const struct vgic_irs_reg_attr gic_v5_irs_regs[] = { + IRS_REG(GICV5_IRS_IDR0), + IRS_REG(GICV5_IRS_IDR1), + IRS_REG(GICV5_IRS_IDR2), + IRS_REG(GICV5_IRS_IDR3), + IRS_REG(GICV5_IRS_IDR4), + IRS_REG(GICV5_IRS_IDR5), + IRS_REG(GICV5_IRS_IDR6), + IRS_REG(GICV5_IRS_IDR7), + IRS_REG(GICV5_IRS_IIDR), + IRS_REG(GICV5_IRS_AIDR), + IRS_REG(GICV5_IRS_CR0), + IRS_REG(GICV5_IRS_CR1), + IRS_REG(GICV5_IRS_SYNCR), + IRS_REG(GICV5_IRS_SYNC_STATUSR), + IRS_REG(GICV5_IRS_SPI_VMR), + IRS_REG(GICV5_IRS_SPI_SELR), + IRS_REG(GICV5_IRS_SPI_DOMAINR), + IRS_REG(GICV5_IRS_SPI_RESAMPLER), + IRS_REG(GICV5_IRS_SPI_CFGR), + IRS_REG(GICV5_IRS_SPI_STATUSR), + IRS_REG(GICV5_IRS_PE_SELR), + IRS_REG(GICV5_IRS_PE_STATUSR), + IRS_REG(GICV5_IRS_PE_CR0), + IRS_REG(GICV5_IRS_IST_BASER), + IRS_REG(GICV5_IRS_IST_CFGR), + IRS_REG(GICV5_IRS_IST_STATUSR), + IRS_REG(GICV5_IRS_MAP_L2_ISTR), + IRS_REG(GICV5_IRS_VMT_BASER), + IRS_REG(GICV5_IRS_VMT_CFGR), + IRS_REG(GICV5_IRS_VMT_STATUSR), + IRS_REG(GICV5_IRS_VPE_SELR), + IRS_REG(GICV5_IRS_VPE_DBR), + IRS_REG(GICV5_IRS_VPE_HPPIR), + IRS_REG(GICV5_IRS_VPE_CR0), + IRS_REG(GICV5_IRS_VPE_STATUSR), + IRS_REG(GICV5_IRS_VM_DBR), + IRS_REG(GICV5_IRS_VM_SELR), + IRS_REG(GICV5_IRS_VM_STATUSR), + IRS_REG(GICV5_IRS_VMAP_L2_VMTR), + IRS_REG(GICV5_IRS_VMAP_VMR), + IRS_REG(GICV5_IRS_VMAP_VISTR), + IRS_REG(GICV5_IRS_VMAP_L2_VISTR), + IRS_REG(GICV5_IRS_VMAP_VPER), + IRS_REG(GICV5_IRS_SAVE_VMR), + IRS_REG(GICV5_IRS_SAVE_VM_STATUSR), + IRS_REG(GICV5_IRS_MEC_IDR), + IRS_REG(GICV5_IRS_MEC_MECID_R), + IRS_REG(GICV5_IRS_MPAM_IDR), + IRS_REG(GICV5_IRS_MPAM_PARTID_R), + IRS_REG(GICV5_IRS_SWERR_STATUSR), + IRS_REG(GICV5_IRS_SWERR_SYNDROMER0), + IRS_REG(GICV5_IRS_SWERR_SYNDROMER1), +}; + +static const struct vgic_sysreg_attr gic_v5_cpu_sysregs[] = { + SR(SYS_ICC_ICSR_EL1), + SR(SYS_ICC_PPI_ENABLER0_EL1), + SR(SYS_ICC_PPI_ENABLER1_EL1), + SR(SYS_ICC_PPI_SACTIVER0_EL1), + SR(SYS_ICC_PPI_SACTIVER1_EL1), + SR(SYS_ICC_PPI_SPENDR0_EL1), + SR(SYS_ICC_PPI_SPENDR1_EL1), + SR(SYS_ICC_PPI_PRIORITYR0_EL1), + SR(SYS_ICC_PPI_PRIORITYR1_EL1), + SR(SYS_ICC_PPI_PRIORITYR2_EL1), + SR(SYS_ICC_PPI_PRIORITYR3_EL1), + SR(SYS_ICC_PPI_PRIORITYR4_EL1), + SR(SYS_ICC_PPI_PRIORITYR5_EL1), + SR(SYS_ICC_PPI_PRIORITYR6_EL1), + SR(SYS_ICC_PPI_PRIORITYR7_EL1), + SR(SYS_ICC_PPI_PRIORITYR8_EL1), + SR(SYS_ICC_PPI_PRIORITYR9_EL1), + SR(SYS_ICC_PPI_PRIORITYR10_EL1), + SR(SYS_ICC_PPI_PRIORITYR11_EL1), + SR(SYS_ICC_PPI_PRIORITYR12_EL1), + SR(SYS_ICC_PPI_PRIORITYR13_EL1), + SR(SYS_ICC_PPI_PRIORITYR14_EL1), + SR(SYS_ICC_PPI_PRIORITYR15_EL1), + SR(SYS_ICC_APR_EL1), + SR(SYS_ICC_CR0_EL1), + SR(SYS_ICC_PCR_EL1), +}; + +static void vgic_v5_save_irs_regs(struct vm_gic *v, u64 *regs) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + ret = __kvm_device_attr_get(v->gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + gic_v5_irs_regs[i].attr, + ®s[i]); + TEST_ASSERT(!ret, "GICv5 IRS_REGS save failed for %s", + gic_v5_irs_regs[i].name); + } +} + +static void vgic_v5_restore_irs_regs(struct vm_gic *v, u64 *regs) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + ret = __kvm_device_attr_set(v->gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + gic_v5_irs_regs[i].attr, + ®s[i]); + TEST_ASSERT(!ret, "GICv5 IRS_REGS restore failed for %s", + gic_v5_irs_regs[i].name); + } +} + +static u64 vgic_v5_saved_irs_reg(u64 *regs, u64 attr) +{ + for (int i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + if (gic_v5_irs_regs[i].attr == attr) + return regs[i]; + } + + TEST_FAIL("GICv5 IRS register 0x%lx was not saved", attr); +} + +static size_t vgic_v5_lpi_ist_state_size(u64 *irs_regs) +{ + u64 cfgr = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_CFGR); + u32 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, cfgr); + + return BIT(lpi_id_bits) * sizeof(__u32); +} + +static void test_vgic_v5_addr_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 addr; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Check existing group/attributes */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, gic_v5_irs_region.attr); + + /* check non existing attribute */ + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, -1); + TEST_ASSERT(ret && errno == ENXIO, "attribute not supported"); + + /* get IRS base address before setting*/ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret && addr == (-1ULL), "GICv5 IRS returns VGIC_ADDR_UNDEF"); + + /* misaligned IRS address settings */ + addr = gic_v5_irs_region.alignment / 0x10; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == EINVAL, "GIC IRS base not aligned"); + + /* out of range address */ + addr = max_phys_size; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == E2BIG, "IRS address beyond IPA limit"); + + /* Space for half an IRS (an IRS is: 2 * irs.alignment). */ + addr = max_phys_size - gic_v5_irs_region.alignment; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == E2BIG, + "half of the IRS is beyond IPA limit"); + + /* set IRS base address @0x0*/ + addr = 0x00000; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret, "GICv5 IRS base correctly set"); + + /* get IRS base address */ + addr = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret && addr == 0, "GICv5 IRS base correctly set"); + + /* Attempt to create a second IRS region */ + addr = 0xE0000; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == EEXIST, "GICv5 IRS base set again"); + + vm_gic_destroy(&v); + + /* Try running a VM without ever setting the IRS base addr */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + ret = run_vcpu(vcpu); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS base not set"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_nr_irqs_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u32 nr_irqs; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0); + + /* Before userspace sets NR_IRQS, no SPI count has been selected. */ + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == 0, "GICv5 NR_IRQS defaults to 0 before init"); + + /* Too few SPIs */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS - 1; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS below minimum"); + + /* Not a multiple of 32 */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS + 1; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS not 32-aligned"); + + /* Larger than KVM's supported VGICv5 SPI count */ + nr_irqs = VGIC_V5_MAX_NR_SPIS + 32; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS above maximum"); + + /* Valid custom SPI count */ + nr_irqs = VGIC_V5_MAX_NR_SPIS; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret, "GICv5 NR_IRQS accepts valid custom SPI count"); + + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == VGIC_V5_MAX_NR_SPIS, + "GICv5 NR_IRQS returns SPI count only"); + + /* A second successful configuration attempt must be rejected. */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 NR_IRQS set twice"); + + /* The maximum supported count must also initialize successfully. */ + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + vm_gic_destroy(&v); + + /* If userspace does not set NR_IRQS, init selects the default. */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == VGIC_V5_DEFAULT_NR_SPIS, + "GICv5 NR_IRQS defaults to 32 SPIs after init"); + + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS * 2; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 NR_IRQS set after init"); + + vm_gic_destroy(&v); + +} + +static void test_vgic_v5_irs_regs_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 attr, val; + int ret, i; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* IRS_REGS attributes can be probed before the IRS base is set. */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0); + + /* IRS_REGS are not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IRS_REGS get before init"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IRS_REGS set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Read all supported IRS regs and write the value back. */ + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + attr = gic_v5_irs_regs[i].attr; + ret = __kvm_has_device_attr(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(!ret, "GICv5 IRS_REGS missing %s", + gic_v5_irs_regs[i].name); + + val = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get failed for %s", + gic_v5_irs_regs[i].name); + + ret = __kvm_device_attr_set(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS set failed for %s", + gic_v5_irs_regs[i].name); + } + + /* Check bad offsets */ + attr = 0x10000; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS_REGS accepted bad offset"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS_REGS get bad offset"); + + /* Check alignment for 32-bit and 64-bit IRS regs. */ + attr = GICV5_IRS_IDR0 + 2; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unaligned 32-bit attr"); + + attr = GICV5_IRS_IST_BASER + 4; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unaligned 64-bit attr"); + + /* Check bad user pointers */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IRS_REGS get with bad pointer"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IRS_REGS set with bad pointer"); + + /* ID restore validation rejects unsupported values. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR0 failed"); + val &= ~GICV5_IRS_IDR0_INT_DOM; + val |= FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, + GICV5_IRS_IDR0_INT_DOM_SECURE); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR0 domain"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR0 failed"); + val |= GICV5_IRS_IDR0_SETLPI; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unsupported IDR0"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR1, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR1 failed"); + val |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, 0x7); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR1, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR1"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR2, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR2"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR5, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR5 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR5, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR5"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR6, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR6 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR6, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR6"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR7, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR7 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR7, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR7"); + + /* Status registers read as idle through userspace accessors. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_SYNC_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_SYNC_STATUSR_IDLE, + "GICv5 IRS_REGS SYNC_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_SPI_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_SPI_STATUSR_IDLE, + "GICv5 IRS_REGS SPI_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_PE_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_PE_STATUSR_IDLE, + "GICv5 IRS_REGS PE_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_IST_STATUSR_IDLE, + "GICv5 IRS_REGS IST_STATUSR is not idle"); + + /* + * Userspace restores IST_BASER without allocating or accessing an IST. + * Supply a synthetic, aligned guest address only to verify the register + * round trip. + */ + val = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + VGIC_V5_TEST_IST_BASER_GPA >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + val |= GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS failed to restore IST_BASER"); + + attr = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &attr); + TEST_ASSERT(!ret && attr == val, "GICv5 IRS_REGS IST_BASER restore mismatch"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_ist_attrs(void) +{ + struct kvm_vgic_v5_ist ist_attr; + struct kvm_vcpu *vcpu; + struct vm_gic v; + u32 spi_ist[VGIC_V5_DEFAULT_NR_SPIS]; + u64 attr; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, 0); + + /* Check non-existing attribute */ + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, 1); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST accepted bad attr"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 1, &ist_attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST get accepted bad attr"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 1, &ist_attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST set accepted bad attr"); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + + /* IST save/restore is not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST get before init"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* A VM with SPIs must provide a userspace IST descriptor. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, NULL); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted NULL descriptor"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, NULL); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted NULL descriptor"); + + /* Check bad userspace IST descriptors. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, (void *)1); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IST get with bad descriptor"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, (void *)1); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IST set with bad descriptor"); + + /* Check missing and incorrectly sized SPI IST buffers. */ + ist_attr = vgic_v5_ist_attr(NULL, 0, NULL, 0); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted missing SPI buffer"); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted missing SPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist) - sizeof(__u32), + NULL, 0); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted bad SPI size"); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted bad SPI size"); + + /* LPI storage must be absent when no LPI IST is configured. */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), spi_ist, + sizeof(__u32)); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted unexpected LPI buffer"); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted unexpected LPI buffer"); + + vm_gic_destroy(&v); + + /* IST restore is rejected after the VM has run. */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, guest_code); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 VM before IST restore test"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST restore after run"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_userspace_ppis_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 user_ppis[2]; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT); + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + user_ppis[0] = 0; + user_ppis[1] = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, user_ppis); + TEST_ASSERT(!ret, "GICv5 USERSPACE_PPIS get failed"); + TEST_ASSERT(user_ppis[0] & BIT(GICV5_ARCH_PPI_SW_PPI), + "GICv5 USERSPACE_PPIS does not expose SW_PPI"); + TEST_ASSERT(!user_ppis[1], "GICv5 USERSPACE_PPIS upper word is not zero"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, user_ppis); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 USERSPACE_PPIS set accepted"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 USERSPACE_PPIS get with bad pointer"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_cpu_sysreg_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 attr, val; + int ret, i; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* Check existing group/attribute */ + attr = PACK_SR(SYS_ICC_CR0_EL1); + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, attr); + + /* CPU sysregs are not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 CPU_SYSREGS get before init"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 CPU_SYSREGS set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Read all exposed CPU sysregs and write the value back. */ + for (i = 0; i < ARRAY_SIZE(gic_v5_cpu_sysregs); i++) { + attr = PACK_SR(gic_v5_cpu_sysregs[i].encoding); + ret = __kvm_has_device_attr(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS missing %s", + gic_v5_cpu_sysregs[i].name); + + val = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS get failed for %s", + gic_v5_cpu_sysregs[i].name); + + ret = __kvm_device_attr_set(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS set failed for %s", + gic_v5_cpu_sysregs[i].name); + } + + /* Check non existent GICv3 sysreg */ + attr = PACK_SR(SYS_ICC_CTLR_EL1); + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 CPU_SYSREGS accepted bad sysreg"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == ENOENT, "GICv5 CPU_SYSREGS get bad sysreg"); + + /* Check non existing vCPU */ + attr = PACK_SR(SYS_ICC_CR0_EL1) | + (1ULL << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT); + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 CPU_SYSREGS accepted bad MPIDR"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 CPU_SYSREGS get bad MPIDR"); + + /* Check bad user pointers */ + attr = PACK_SR(SYS_ICC_CR0_EL1); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 CPU_SYSREGS get with bad pointer"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 CPU_SYSREGS set with bad pointer"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_sparse_vcpu_ids(void) +{ + struct kvm_vcpu *vcpus[VGIC_V5_SPARSE_NR_VCPUS]; + const u32 vcpu_ids[VGIC_V5_SPARSE_NR_VCPUS] = { + VGIC_V5_SPARSE_VCPU0_ID, + VGIC_V5_SPARSE_VCPU1_ID, + }; + int ret, i; + struct ucall uc; + struct vm_gic v; + u64 attr; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, VGIC_V5_SPARSE_NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Create two vCPUs with legal, albeit sparse, IDs. */ + for (i = 0; i < ARRAY_SIZE(vcpus); i++) { + vcpus[i] = vm_vcpu_add(v.vm, vcpu_ids[i], guest_iaffid_code); + TEST_ASSERT(vcpus[i], "Failed to create vCPU %u", vcpu_ids[i]); + } + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v.vm); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Check both are runnable, and read back their correct IAFFIDs */ + for (i = 0; i < ARRAY_SIZE(vcpus); i++) { + /* IAFFID is the VPE ID and must match the userspace vCPU ID. */ + ret = run_vcpu(vcpus[i]); + TEST_ASSERT(!ret, "Failed to run GICv5 vCPU %u", vcpu_ids[i]); + TEST_ASSERT(get_ucall(vcpus[i], &uc) == UCALL_SYNC && + uc.args[1] == vcpu_ids[i], + "GICv5 vCPU %u IAFFID mismatch", vcpu_ids[i]); + + ret = run_vcpu(vcpus[i]); + TEST_ASSERT(!ret, "Failed to complete GICv5 vCPU %u", vcpu_ids[i]); + TEST_ASSERT(get_ucall(vcpus[i], NULL) == UCALL_DONE, + "GICv5 vCPU %u did not complete", vcpu_ids[i]); + } + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; struct ucall uc; u64 user_ppis[2]; struct vm_gic v; + uint64_t attr; int ret, i; v.gic_dev_type = gic_dev_type; @@ -111,11 +1597,17 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) vcpus[i] = vm_vcpu_add(v.vm, i, guest_code); vm_init_descriptor_tables(v.vm); - vm_install_exception_handler(v.vm, VECTOR_IRQ_CURRENT, guest_irq_handler); + vm_install_exception_handler(v.vm, VECTOR_IRQ_CURRENT, + guest_ppi_irq_handler); for (i = 0; i < NR_VCPUS; i++) vcpu_init_descriptor_tables(vcpus[i]); + /* Set the address of the IRS before initialising the GIC */ + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); @@ -127,6 +1619,20 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) TEST_ASSERT(user_ppis[0] & BIT(GICV5_ARCH_PPI_SW_PPI), "SW_PPI is not drivable by userspace"); + /* PPIs not explicitly exposed to userspace must be rejected. */ + for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) { + if (user_ppis[i / 64] & BIT_ULL(i % 64)) + continue; + + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_PPI, i, 1); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted non-userspace PPI %d", i); + } + + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_PPI, + VGIC_V5_NR_PRIVATE_IRQS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted out-of-range PPI"); + while (1) { ret = run_vcpu(vcpus[0]); if (ret) @@ -148,9 +1654,9 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) kvm_irq_line(v.vm, irq, level); } else if (uc.args[1] == GUEST_CMD_IS_AWAKE) { - pr_info("Guest skipping WFI due to pending IRQ\n"); + pr_debug("Guest skipping WFI due to pending IRQ\n"); } else if (uc.args[1] == GUEST_CMD_IRQ_CDIA) { - pr_info("Guest acknowledged IRQ\n"); + pr_debug("Guest acknowledged IRQ\n"); } continue; @@ -170,6 +1676,304 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) vm_gic_destroy(&v); } +static void test_vgic_v5_spis(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + int ret; + + /* Default NR_IRQS exposes 32 SPIs, numbered 0..31 in KVM_IRQ_LINE. */ + vgic_v5_spi_line_vm_create(&v, &vcpu, VGIC_V5_DEFAULT_NR_SPIS); + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_SPI, + VGIC_V5_DEFAULT_NR_SPIS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted first invalid SPI"); + + vm_gic_destroy(&v); + + /* Basic SPI injection through KVM_IRQ_LINE. */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 0, false, false); + + /* The last valid SPI in the maximum configured range is injectable. */ + vgic_v5_run_spi_line_test(VGIC_V5_MAX_NR_SPIS, + VGIC_V5_MAX_NR_SPIS - 1, false, false); + + vgic_v5_spi_line_vm_create(&v, &vcpu, VGIC_V5_MAX_NR_SPIS); + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_SPI, + VGIC_V5_MAX_NR_SPIS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted configured invalid SPI"); + vm_gic_destroy(&v); + + /* + * Edge SPIs remain pending after the line is lowered. The low + * transition is injected before the guest runs and must be ignored: + * once an edge interrupt is pending, lowering the line cannot recall it. + */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 1, false, true); + + /* Level SPIs can be raised and delivered through KVM_IRQ_LINE. */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 2, true, false); +} + +static void test_vgic_v5_lpis(void) +{ + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]; + struct ucall uc; + struct vm_gic v; + int ret; + + lpi_ist_ready = false; + vgic_v5_lpi_vm_create(&v, vcpus); + sync_global_to_guest(v.vm, lpi_ist_ready); + + /* VPE0 programs a linear LPI IST from the virtual IRS ID registers. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 LPI vCPU0 did not become ready"); + + /* VPE1 waits for the shared LPI setup and enables its CPU interface. */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU1"); + TEST_ASSERT(get_ucall(vcpus[1], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 LPI vCPU1 did not become ready"); + + /* VPE0 sends an LPI to VPE1. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0 sender"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_LPI_SENT, + "GICv5 LPI vCPU0 did not send LPI"); + + /* VPE1 consumes the LPI and replies with another LPI. */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU1 receiver"); + TEST_ASSERT(get_ucall(vcpus[1], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_LPI_REPLIED, + "GICv5 LPI vCPU1 did not reply"); + + /* VPE0 consumes the reply LPI and ends the test. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0 receiver"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_DONE, + "GICv5 LPI vCPU0 did not complete"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_ist_save_restore(void) +{ + static u32 lpi_ist[VGIC_V5_LPI_IST_SIZE / sizeof(u32)]; + static u32 lpi_ist_resave[VGIC_V5_LPI_IST_SIZE / sizeof(u32)]; + static u32 spi_ist[VGIC_V5_DEFAULT_NR_SPIS]; + static u32 spi_ist_resave[VGIC_V5_DEFAULT_NR_SPIS]; + static u64 irs_regs[ARRAY_SIZE(gic_v5_irs_regs)]; + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]; + struct kvm_vgic_v5_ist ist_attr; + struct vm_gic src, dst; + u64 ist_baser, bad_ist_baser; + u64 ist_cfgr, bad_ist_cfgr; + size_t lpi_ist_size; + int ret; + + memset(ist_restore_irq_count, 0, sizeof(ist_restore_irq_count)); + + vgic_v5_ist_restore_vm_create(&src, vcpus, guest_ist_source_code, + guest_unexpected_irq_handler); + + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST save source VM"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IST_SOURCE_READY, + "GICv5 IST source guest did not become ready"); + + /* + * The guest configured enable/priority/affinity state. Make selected + * SPIs pending from userspace so the SPI IST save path sees them. + */ + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PRIO_HIGH, 1); + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PRIO_LOW, 1); + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PENDING_DISABLED, 1); + + vgic_v5_save_irs_regs(&src, irs_regs); + lpi_ist_size = vgic_v5_lpi_ist_state_size(irs_regs); + + /* + * VCPU0 has configured the initial state. KVM must reject missing or + * incorrectly sized LPI storage before accepting the complete IST image. + */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST save accepted missing LPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size - sizeof(__u32)); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST save accepted bad LPI size"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST save failed"); + + vm_gic_destroy(&src); + + /* Create the guest that we are restoring state into */ + memset(ist_restore_irq_count, 0, sizeof(ist_restore_irq_count)); + vgic_v5_ist_restore_vm_create(&dst, vcpus, guest_ist_restore_code, + guest_ist_restore_irq_handler); + sync_global_to_guest(dst.vm, ist_restore_irq_count); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore before IRS state"); + + vgic_v5_restore_irs_regs(&dst, irs_regs); + + /* BASER remains mutable until the host LPI IST has been allocated. */ + ist_baser = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_BASER); + bad_ist_baser = ist_baser ^ + FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected changed unallocated IST_BASER"); + + bad_ist_baser = ist_baser & ~GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected invalidated unallocated IST_BASER"); + + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &ist_baser); + TEST_ASSERT(!ret, "GICv5 failed to restore IST_BASER"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore accepted missing LPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size - sizeof(__u32)); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore accepted bad LPI size"); + + /* Restore the VMM-owned SPI and LPI IST images. */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST restore failed"); + + /* A live IST makes BASER and the guest-visible configuration immutable. */ + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected unchanged live IST_BASER"); + + bad_ist_baser = ist_baser ^ + FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted changed live IST_BASER"); + + bad_ist_baser = ist_baser & ~GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted invalidated live IST_BASER"); + + ist_cfgr = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_CFGR); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_CFGR, &ist_cfgr); + TEST_ASSERT(!ret, "GICv5 rejected unchanged live IST_CFGR"); + + bad_ist_cfgr = ist_cfgr ^ + FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_CFGR, &bad_ist_cfgr); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted changed live IST_CFGR"); + + /* + * Re-save the restored IST image before the destination VM runs, so a + * dropped serialized field fails independently of later interrupt + * delivery. + */ + ist_attr = vgic_v5_ist_attr(spi_ist_resave, sizeof(spi_ist_resave), + lpi_ist_resave, lpi_ist_size); + ret = __kvm_device_attr_get(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST re-save failed"); + TEST_ASSERT(!memcmp(spi_ist, spi_ist_resave, sizeof(spi_ist)), + "GICv5 SPI IST state changed across restore"); + TEST_ASSERT(!memcmp(lpi_ist, lpi_ist_resave, lpi_ist_size), + "GICv5 LPI IST state changed across restore"); + + /* + * At this stage, the guest's IST state should match what was saved. Run + * each vcpu once to let it enable interrupts, etc. + */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST restore vCPU0"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IS_READY, + "GICv5 IST restore vCPU0 did not become ready"); + + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST restore vCPU1"); + vgic_v5_expect_sync(vcpus[1], GUEST_CMD_IS_READY, + "GICv5 IST restore vCPU1 did not become ready"); + + /* + * VPE0 consumes two pending SPIs in restored priority order, then + * enables a restored pending-but-disabled SPI. + */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to consume restored GICv5 SPIs"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IST_SPI_INJECT_READY, + "GICv5 IST restore vCPU0 did not consume restored SPIs"); + + /* + * This SPI was enabled before save, but not pending. Inject it after + * restore to prove enable, priority and affinity were restored. + */ + vgic_v5_spi_line(dst.vm, RESTORE_TEST_SPI_ENABLED, 1); + + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to consume post-restore GICv5 SPI"); + TEST_ASSERT(get_ucall(vcpus[0], NULL) == UCALL_DONE, + "GICv5 IST restore vCPU0 did not complete"); + + /* + * VPE1 consumes a restored pending LPI, then pends another LPI that + * was enabled before save but not pending. + */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to consume restored GICv5 LPI"); + vgic_v5_expect_sync(vcpus[1], GUEST_CMD_IST_LPI_INJECT_READY, + "GICv5 IST restore vCPU1 did not consume restored LPI"); + + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to consume post-restore GICv5 LPI"); + TEST_ASSERT(get_ucall(vcpus[1], NULL) == UCALL_DONE, + "GICv5 IST restore vCPU1 did not complete"); + + vm_gic_destroy(&dst); +} + /* * Returns 0 if it's possible to create GIC device of a given type (V5). */ @@ -201,13 +2005,46 @@ int test_kvm_device(u32 gic_dev_type) void run_tests(u32 gic_dev_type) { + pr_info("Test VGICv5 address attrs\n"); + test_vgic_v5_addr_attrs(); + + pr_info("Test VGICv5 NR_IRQS attrs\n"); + test_vgic_v5_nr_irqs_attrs(); + + pr_info("Test VGICv5 IRS_REGS attrs\n"); + test_vgic_v5_irs_regs_attrs(); + + pr_info("Test VGICv5 IST attrs\n"); + test_vgic_v5_ist_attrs(); + + pr_info("Test VGICv5 userspace PPI attrs\n"); + test_vgic_v5_userspace_ppis_attrs(); + + pr_info("Test VGICv5 CPU sysreg attrs\n"); + test_vgic_v5_cpu_sysreg_attrs(); + + pr_info("Test VGICv5 sparse vCPU IDs\n"); + test_vgic_v5_sparse_vcpu_ids(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); + + pr_info("Test VGICv5 SPIs\n"); + test_vgic_v5_spis(); + + pr_info("Test VGICv5 LPIs\n"); + test_vgic_v5_lpis(); + + pr_info("Test VGICv5 IST save/restore\n"); + test_vgic_v5_ist_save_restore(); } int main(int ac, char **av) { - int ret; + int pa_bits, ret; + + pa_bits = vm_guest_mode_params[VM_MODE_DEFAULT].pa_bits; + max_phys_size = 1ULL << pa_bits; test_disable_default_vgic();
diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index f8b3d0b..619a0b2 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c
@@ -297,8 +297,7 @@ int main(int argc, char *argv[]) p.single_uffd = true; break; case 'd': - p.uffd_delay = strtoul(optarg, NULL, 0); - TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported."); + p.uffd_delay = atoi_non_negative("UFFD delay", optarg); break; case 'b': guest_percpu_mem_size = parse_size(optarg);
diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index af5eb03..8189eaa 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c
@@ -44,8 +44,11 @@ #define TEST_MIN_WRITES_PER_ITERATION 0x100 /* Dirty bitmaps are always little endian, so we need to swap on big endian */ -#if defined(__s390x__) +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) +#else +# define BITOP_LE_SWIZZLE 0 +#endif # define test_bit_le(nr, addr) \ test_bit((nr) ^ BITOP_LE_SWIZZLE, addr) # define __set_bit_le(nr, addr) \ @@ -56,13 +59,6 @@ __test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, addr) # define __test_and_clear_bit_le(nr, addr) \ __test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr) -#else -# define test_bit_le test_bit -# define __set_bit_le __set_bit -# define __clear_bit_le __clear_bit -# define __test_and_set_bit_le __test_and_set_bit -# define __test_and_clear_bit_le __test_and_clear_bit -#endif #define TEST_DIRTY_RING_COUNT 65536
diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index 2233d87..cd5df88 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c
@@ -80,7 +80,7 @@ static void test_mbind(int fd, size_t total_size) { const unsigned long nodemask_0 = 1; /* nid: 0 */ unsigned long nodemask = 0; - unsigned long maxnode = BITS_PER_TYPE(nodemask); + unsigned long maxnode = MAXNODE_FOR_MASK(nodemask); int policy; char *mem; int ret;
diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c index 1c20892..ef004af 100644 --- a/tools/testing/selftests/kvm/hardware_disable_test.c +++ b/tools/testing/selftests/kvm/hardware_disable_test.c
@@ -130,6 +130,9 @@ int main(int argc, char **argv) int s, r, cpu, i; pid_t pid; + /* PowerPC PAPR L0 cannot guarantee 512 overlapping H_GUEST_CREATE from fork+SIGKILL. */ + TEST_REQUIRE(!kvm_is_ppc_pseries_lpar()); + kvm_sched_getaffinity(0, sizeof(cpu_set_t), &allowed_cpu_set); for (i = 0; i < NR_VCPUS && CPU_COUNT(&allowed_cpu_set); i++) {
diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index eb523d9..2d45551 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h
@@ -10,6 +10,106 @@ #include "processor.h" +/* GIC component base address is guest PA space */ +#define GICV5_IRS_CONFIG_BASE_GPA 0x8000000ULL +#define GICV5_IRS_CONFIG_BASE_GVA ((volatile void *)GICV5_IRS_CONFIG_BASE_GPA) + +#define GICV5_IRS_SIZE 0x20000 +#define GICV5_IRS_ALIGN 0x10000 + +/* + * IRS register offsets. Keep this selftest copy scoped to the offsets and + * fields exercised by the userspace API tests. + */ +#define GICV5_IRS_IDR0 0x0000 +#define GICV5_IRS_IDR1 0x0004 +#define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 +#define GICV5_IRS_IDR5 0x0014 +#define GICV5_IRS_IDR6 0x0018 +#define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 +#define GICV5_IRS_CR0 0x0080 +#define GICV5_IRS_CR1 0x0084 +#define GICV5_IRS_SYNCR 0x00c0 +#define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 +#define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 +#define GICV5_IRS_SPI_CFGR 0x0114 +#define GICV5_IRS_SPI_STATUSR 0x0118 +#define GICV5_IRS_PE_SELR 0x0140 +#define GICV5_IRS_PE_STATUSR 0x0144 +#define GICV5_IRS_PE_CR0 0x0148 +#define GICV5_IRS_IST_BASER 0x0180 +#define GICV5_IRS_IST_CFGR 0x0190 +#define GICV5_IRS_IST_STATUSR 0x0194 +#define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 + +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 + +#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) + +#define GICV5_IRS_IDR2_ISTMD_SZ GENMASK(19, 15) +#define GICV5_IRS_IDR2_ISTMD BIT(14) +#define GICV5_IRS_IDR2_IST_L2SZ GENMASK(13, 11) +#define GICV5_IRS_IDR2_IST_LEVELS BIT(10) +#define GICV5_IRS_IDR2_MIN_LPI_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR2_LPI BIT(5) +#define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_CR0_IDLE BIT(1) +#define GICV5_IRS_CR0_IRSEN BIT(0) + +#define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_CFGR_TM BIT(0) + +#define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_PE_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL +#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) +#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5) +#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00 +#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01 +#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10 + /* * Definitions for GICv5 instructions for the Current Domain */ @@ -77,6 +177,11 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE GENMASK_ULL(31, 29) +#define GICV5_HWIRQ_ID GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE_LPI 0x2 +#define GICV5_HWIRQ_TYPE_SPI 0x3 + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn)
diff --git a/tools/testing/selftests/kvm/include/arm64/processor.h b/tools/testing/selftests/kvm/include/arm64/processor.h index b8a902b..5878a6a 100644 --- a/tools/testing/selftests/kvm/include/arm64/processor.h +++ b/tools/testing/selftests/kvm/include/arm64/processor.h
@@ -12,6 +12,7 @@ #include <linux/stringify.h> #include <linux/types.h> +#include <asm/barrier.h> #include <asm/brk-imm.h> #include <asm/esr.h> #include <asm/sysreg.h> @@ -191,11 +192,8 @@ static inline void cpu_relax(void) #define dsb(opt) asm volatile("dsb " #opt : : : "memory") #define dmb(opt) asm volatile("dmb " #opt : : : "memory") -#define dma_wmb() dmb(oshst) #define __iowmb() dma_wmb() -#define dma_rmb() dmb(oshld) - #define __iormb(v) \ ({ \ unsigned long tmp; \
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 5a7a455..e261887 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -14,6 +14,7 @@ #include <linux/kvm.h> #include "linux/rbtree.h" #include <linux/types.h> +#include <unistd.h> #include <asm/atomic.h> #include <asm/kvm.h> @@ -33,6 +34,8 @@ #define NSEC_PER_SEC 1000000000L +#define KVM_INVALID_MEMSLOT UINT_MAX + struct userspace_mem_region { struct kvm_userspace_memory_region2 region; struct sparsebit *unused_phy_pages; @@ -80,11 +83,45 @@ struct userspace_mem_regions { DECLARE_HASHTABLE(slot_hash, 9); }; +/* + * Memory region types are passed to various page allocators to communicate + * various properties and metadata related to the allocation. Note, the + * descriptions below described the primary usage of each type. Individual + * tests may allocate memory for other purposes. + * + * By default, all regions except TEST_EXTRA are mapped to memslot '0'. The + * TEST_EXTRA region is left unmapped as it's intended to be used only for test + * specific allocations, i.e. should never be used by core/infrastructure code. + * Tests can override the memslot for any or all types, e.g. so that all test + * data is allocated from a curated memslot. + */ enum kvm_mem_region_type { + /* + * The CODE region is used by lib/elf when loading the test's code into + * guest memory. + */ MEM_REGION_CODE, + /* + * The DATA region is used to allocate core data structures, e.g. vCPU + * stacks, VM exception tables, x86's TSS, etc. + */ MEM_REGION_DATA, + /* + * The PT region, a.k.a. Page Table region, is used to allocate page + * table pages. + */ MEM_REGION_PT, + /* + * The TEST_DATA region is used for allocating test data that is either + * test specific, and/or isn't considered a "core" data structure. + */ MEM_REGION_TEST_DATA, + /* + * The TEST_EXTRA region is for special snowflakes, where a test wants + * to create and use a one-off memslot, without impacting "normal" test + * data allocations. + */ + MEM_REGION_TEST_EXTRA, NR_MEM_REGIONS, }; @@ -127,11 +164,6 @@ struct kvm_vm { struct kvm_binary_stats stats; - /* - * KVM region slots. These are the default memslots used by page - * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] - * memslot. - */ u32 memslots[NR_MEM_REGIONS]; }; @@ -169,7 +201,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot); static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, enum kvm_mem_region_type type) { - assert(type < NR_MEM_REGIONS); + TEST_ASSERT(type < NR_MEM_REGIONS, + "Invalid memory region type '%u'", type); return memslot2region(vm, vm->memslots[type]); } @@ -209,6 +242,9 @@ enum vm_guest_mode { VM_MODE_P41V48_4K, VM_MODE_P41V39_4K, + VM_MODE_P52V52_4K, /* For powerpc64 */ + VM_MODE_P52V52_64K, + NUM_VM_MODES, }; @@ -268,6 +304,12 @@ extern enum vm_guest_mode vm_mode_default; #define MIN_PAGE_SHIFT 12U #define ptes_per_page(page_size) ((page_size) / 8) +#elif defined(__powerpc64__) + +#define VM_MODE_DEFAULT vm_mode_default +#define MIN_PAGE_SHIFT 12U +#define ptes_per_page(page_size) ((page_size) / 8) + #endif #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) @@ -392,6 +434,16 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ }) +#define __gmem_ioctl(gmem_fd, cmd, arg) \ + kvm_do_ioctl(gmem_fd, cmd, arg) + +#define gmem_ioctl(gmem_fd, cmd, arg) \ +({ \ + int ret = __gmem_ioctl(gmem_fd, cmd, arg); \ + \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ +}) + /* * Looks up and returns the value corresponding to the capability * (KVM_CAP_*) given by cap. @@ -418,8 +470,19 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes); + +/* + * KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These + * flows need significant enhancements to support multiple attributes. + */ +#define TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes) \ + TEST_ASSERT(!(attributes) || (attributes) == KVM_MEMORY_ATTRIBUTE_PRIVATE, \ + "Update me to support multiple attributes!") + static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, - u64 size, u64 attributes) + size_t size, u64 attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -428,27 +491,109 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, .flags = 0, }; - /* - * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows - * need significant enhancements to support multiple attributes. - */ - TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, - "Update me to support multiple attributes!"); + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); } +static inline int __gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes, + u64 *error_offset) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + .error_offset = 0, + }; + int r; + + r = __gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); + + /* Copy error_offset regardless of r so caller can check. */ + if (error_offset) + *error_offset = attr.error_offset; + + return r; +} + +static inline int __gmem_set_private(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + error_offset); +} + +static inline int __gmem_set_shared(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, 0, + error_offset); +} + +static inline void gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + }; + + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); + + __TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) > 0, + "No valid attributes for guest_memfd ioctl!"); + + gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); +} + +static inline void gmem_set_private(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void gmem_set_shared(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, 0); +} + +static inline void vm_mem_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, + size_t size, u64 attrs) +{ + if (kvm_has_gmem_attributes) { + gpa_t end = gpa + size; + off_t fd_offset; + gpa_t addr; + size_t len; + int fd; + + for (addr = gpa; addr < end; addr += len) { + fd = kvm_gpa_to_guest_memfd(vm, addr, &fd_offset, &len); + len = min(end - addr, len); + + gmem_set_memory_attributes(fd, fd_offset, len, attrs); + } + } else { + vm_set_memory_attributes(vm, gpa, size, attrs); + } +} static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); + vm_mem_set_memory_attributes(vm, gpa, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); } static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, 0); + vm_mem_set_memory_attributes(vm, gpa, size, 0); } void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size, @@ -690,17 +835,47 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva); void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd_fd, u64 guest_memfd_offset); + int gmem_fd, u64 gmem_offset, u64 gmem_flags); + + +static inline void ____vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + u32 slot) +{ + TEST_ASSERT(vm->memslots[type] == KVM_INVALID_MEMSLOT, + "Memory region type '%u' was already overridden with slot=%u", + type, vm->memslots[type]); + + vm->memslots[type] = slot; +} + +static inline void __vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages, + u32 flags) +{ + ____vm_override_mem_region(vm, type, slot); + vm_userspace_mem_region_add(vm, src_type, gpa, slot, npages, flags); +} + +static inline void vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages) +{ + __vm_override_mem_region(vm, type, src_type, gpa, slot, npages, 0); +} #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) @@ -732,6 +907,11 @@ void *addr_gva2hva(struct kvm_vm *vm, gva_t gva); gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva); void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa); +static inline int kvm_slot_to_fd(struct kvm_vm *vm, u32 slot) +{ + return memslot2region(vm, slot)->fd; +} + #ifndef vcpu_arch_put_guest #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) #endif @@ -990,23 +1170,57 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot); -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, gpa_t min_gpa, - u32 memslot, bool protected); -gpa_t vm_alloc_page_table(struct kvm_vm *vm); +bool kvm_arch_needs_naturally_aligned_page_tables(void); -static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, - gpa_t min_gpa, u32 memslot) +/* + * True on powerpc when this host is a pseries LPAR (nested case). + * PowerPC's PAPR L0 does not guarantee H_GUEST_CREATE / H_GUEST_CREATE_VCPU up + * to MAX_VCPUS or creating hundreds of VMs - due to limited resouces at L0 for + * guest management space. This helper allows us to skip such tests when we are + * running nested. + */ +static inline bool kvm_is_ppc_pseries_lpar(void) +{ +#ifdef __powerpc64__ + return !access("/proc/device-tree/rtas/ibm,hypertas-functions", F_OK); +#else + return false; +#endif +} + +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned); +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type, bool protected); + +static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type) { /* * By default, allocate memory as protected for VMs that support * protected memory, as the majority of memory for such VMs is * protected, i.e. using shared memory is effectively opt-in. */ - return __vm_phy_pages_alloc(vm, num, min_gpa, memslot, + return __vm_phy_pages_alloc(vm, nr_pages, type, vm_arch_has_protected_memory(vm)); } +static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, + enum kvm_mem_region_type type) +{ + return vm_phy_pages_alloc(vm, 1, type); +} + +static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages) +{ + return vm_phy_pages_alloc(vm, nr_pages, MEM_REGION_PT); +} + +static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) +{ + return vm_alloc_page_table_pages(vm, 1); +} + /* * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also * loads the test binary into guest memory and creates an IRQ chip (x86 only). @@ -1123,6 +1337,9 @@ static inline int pin_self_to_any_cpu(void) return pin_task_to_any_cpu(pthread_self()); } +void pin_pages(void *vaddr, uint64_t size); +void unpin_pages(void); + void kvm_print_vcpu_pinning_help(void); void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], int nr_vcpus);
diff --git a/tools/testing/selftests/kvm/include/loongarch/processor.h b/tools/testing/selftests/kvm/include/loongarch/processor.h index 0e59daa..8d9fabd 100644 --- a/tools/testing/selftests/kvm/include/loongarch/processor.h +++ b/tools/testing/selftests/kvm/include/loongarch/processor.h
@@ -79,6 +79,7 @@ #define CPUCFG2_LASX BIT(7) #define CPUCFG2_LLFTP BIT(14) #define CPUCFG2_LLFTPREV GENMASK(17, 15) +#define CPUCFG2_PTW BIT(24) /* LoongArch Basic CSR registers */ #define LOONGARCH_CSR_CRMD 0x0 /* Current mode info */ @@ -128,6 +129,8 @@ #define LOONGARCH_CSR_PGD 0x1b #define LOONGARCH_CSR_PWCTL0 0x1c #define LOONGARCH_CSR_PWCTL1 0x1d +#define CSR_PWCTL1_PTW_SHIFT 24 +#define CSR_PWCTL1_PTW BIT_ULL(CSR_PWCTL1_PTW_SHIFT) #define LOONGARCH_CSR_STLBPGSIZE 0x1e #define LOONGARCH_CSR_CPUID 0x20 #define LOONGARCH_CSR_KS0 0x30
diff --git a/tools/testing/selftests/kvm/include/numaif.h b/tools/testing/selftests/kvm/include/numaif.h index 29572a6..0945500 100644 --- a/tools/testing/selftests/kvm/include/numaif.h +++ b/tools/testing/selftests/kvm/include/numaif.h
@@ -6,6 +6,7 @@ #include <dirent.h> +#include <linux/bitmap.h> #include <linux/mempolicy.h> #include "kvm_syscalls.h" @@ -30,6 +31,48 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode, const unsigned long *, nodemask, unsigned long, maxnode, unsigned int, flags); +/* + * Calculate the @maxnode param for the above syscalls given the mask that will + * be passed to the kernel, to account for a longstanding off-by-one bug in the + * kernel that isn't properly documented in the manpages. The manpages say + * that @maxnode is "the maximum node ID plus one", but the kernel's actual + * behavior is "the number of bits in the mask plus one", i.e. "the maximum + * node ID plus two". + */ +#define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1) + +static inline int kvm_get_numa_memory_nodes(unsigned long *nodemask) +{ + int r; + + *nodemask = 0; + + r = get_mempolicy(NULL, nodemask, MAXNODE_FOR_MASK(*nodemask), 0, + MPOL_F_MEMS_ALLOWED); + TEST_ASSERT(!r || errno == ENOSYS || errno == EPERM, + "Unexpected get_mempolicy() failure"); + return __builtin_popcountl(*nodemask); +} + +/* + * Return the node ID of the next NUMA node in the mask, starting at @from+1. + * Guarantees a node is found, and that the found node is not @from. Pass -1 + * to find the first node in the mask. + */ +static inline int kvm_get_next_numa_node(unsigned long nodemask, int from) +{ + const unsigned long nr_bits = BITS_PER_TYPE(nodemask); + int to; + + to = find_next_bit(&nodemask, nr_bits, from + 1); + if (to == nr_bits) + to = find_next_bit(&nodemask, nr_bits, 0); + + TEST_ASSERT(to != nr_bits && to != from, + "Unabled to find second NUMA node (from = %d, to = %d)", from, to); + return to; +} + static inline int get_max_numa_node(void) { struct dirent *de;
diff --git a/tools/testing/selftests/kvm/include/powerpc/hcall.h b/tools/testing/selftests/kvm/include/powerpc/hcall.h new file mode 100644 index 0000000..21b2bf9 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/hcall.h
@@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc hcall defines + */ +#ifndef SELFTEST_KVM_HCALL_H +#define SELFTEST_KVM_HCALL_H + +#include <linux/compiler.h> + +/* Ucalls use unimplemented PAPR hcall 0 which exits KVM */ +#define H_UCALL 0 + +s64 hcall0(u64 token); +s64 hcall1(u64 token, u64 arg1); +s64 hcall2(u64 token, u64 arg1, u64 arg2); + +#endif
diff --git a/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h new file mode 100644 index 0000000..5d45c25 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h
@@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UTIL_ARCH_H +#define SELFTEST_KVM_UTIL_ARCH_H + +#include <stdint.h> + +#include "kvm_util_types.h" + +struct kvm_mmu_arch {}; + +/* Page table fragment cache for guest page tables < page size */ +struct vm_pt_frag_cache { + gpa_t page; + size_t page_nr_used; +}; + +struct kvm_vm_arch { + gpa_t prtb; /* process table */ + struct vm_pt_frag_cache pt_frag_cache[2]; /* 256B and 4KB PT caches */ +}; + +#endif /* SELFTEST_KVM_UTIL_ARCH_H */
diff --git a/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h new file mode 100644 index 0000000..b9df646 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h
@@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc asm specific defines + */ +#ifndef SELFTEST_KVM_PPC_ASM_H +#define SELFTEST_KVM_PPC_ASM_H + +#define STACK_FRAME_MIN_SIZE 112 /* Could be 32 on ELFv2 */ +#define STACK_REDZONE_SIZE 512 + +#define INT_FRAME_SIZE (STACK_FRAME_MIN_SIZE + STACK_REDZONE_SIZE) + +#define SPR_SRR0 0x01a +#define SPR_SRR1 0x01b +#define SPR_CFAR 0x01c + +#define MSR_SF 0x8000000000000000ULL +#define MSR_HV 0x1000000000000000ULL +#define MSR_VEC 0x0000000002000000ULL +#define MSR_VSX 0x0000000000800000ULL +#define MSR_EE 0x0000000000008000ULL +#define MSR_PR 0x0000000000004000ULL +#define MSR_FP 0x0000000000002000ULL +#define MSR_ME 0x0000000000001000ULL +#define MSR_IR 0x0000000000000020ULL +#define MSR_DR 0x0000000000000010ULL +#define MSR_RI 0x0000000000000002ULL +#define MSR_LE 0x0000000000000001ULL + +#define LPCR_ILE 0x0000000002000000ULL + +#endif
diff --git a/tools/testing/selftests/kvm/include/powerpc/processor.h b/tools/testing/selftests/kvm/include/powerpc/processor.h new file mode 100644 index 0000000..f0ee21a --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/processor.h
@@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc processor specific defines + */ +#ifndef SELFTEST_KVM_PROCESSOR_H +#define SELFTEST_KVM_PROCESSOR_H + +#include <linux/compiler.h> +#include "ppc_asm.h" +#include "kvm_util_types.h" + +extern unsigned char __interrupts_start[]; +extern unsigned char __interrupts_end[]; + +struct kvm_vm; +struct kvm_vcpu; + +struct ex_regs { + u64 gprs[32]; + u64 nia; + u64 msr; + u64 cfar; + u64 lr; + u64 ctr; + u64 xer; + u32 cr; + u32 trap; + gva_t gva; /* gva of this struct */ +}; + +void vm_install_exception_handler(struct kvm_vm *vm, int vector, + void (*handler)(struct ex_regs *)); + +static inline void cpu_relax(void) +{ + asm volatile("" ::: "memory"); +} + +#endif
diff --git a/tools/testing/selftests/kvm/include/powerpc/ucall.h b/tools/testing/selftests/kvm/include/powerpc/ucall.h new file mode 100644 index 0000000..e0dbe91 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ucall.h
@@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UCALL_H +#define SELFTEST_KVM_UCALL_H + +#include "hcall.h" + +#define UCALL_EXIT_REASON KVM_EXIT_PAPR_HCALL + +#define UCALL_R4_UCALL 0x5715 /* regular ucall, r5 contains ucall pointer */ +#define UCALL_R4_SIMPLE 0x0000 /* simple exit usable by asm with no ucall data */ + +static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa) +{ +} + +static inline void ucall_arch_do_ucall(gva_t uc) +{ + hcall2(H_UCALL, UCALL_R4_UCALL, (uintptr_t)(uc)); +} + +#endif
diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a6a3e16..c80019e 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h
@@ -49,10 +49,6 @@ do { \ #define TEST_REQUIRE(f) __TEST_REQUIRE(f, "Requirement not met: %s", #f) -ssize_t test_write(int fd, const void *buf, size_t count); -ssize_t test_read(int fd, void *buf, size_t count); -int test_seq_read(const char *path, char **bufp, size_t *sizep); - void __printf(5, 6) test_assert(bool exp, const char *exp_str, const char *file, unsigned int line, const char *fmt, ...); @@ -83,21 +79,23 @@ do { \ __builtin_unreachable(); \ } while (0) -extern sigjmp_buf expect_sigbus_jmpbuf; -void expect_sigbus_handler(int signum); +extern __thread sigjmp_buf expect_sigbus_jmpbuf; +extern __thread volatile sig_atomic_t expecting_sigbus; +void catchall_signal_handler(int signum); -#define TEST_EXPECT_SIGBUS(action) \ -do { \ - struct sigaction sa_old, sa_new = { \ - .sa_handler = expect_sigbus_handler, \ - }; \ - \ - sigaction(SIGBUS, &sa_new, &sa_old); \ - if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ - action; \ - TEST_FAIL("'%s' should have triggered SIGBUS", #action); \ - } \ - sigaction(SIGBUS, &sa_old, NULL); \ +#define TEST_EXPECT_SIGBUS(action) \ +do { \ + struct sigaction __sa = {}; \ + \ + TEST_ASSERT_EQ(sigaction(SIGBUS, NULL, &__sa), 0); \ + TEST_ASSERT_EQ(__sa.sa_handler, &catchall_signal_handler); \ + \ + expecting_sigbus = true; \ + if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ + action; \ + TEST_FAIL("'%s' should have triggered SIGBUS", #action);\ + } \ + expecting_sigbus = false; \ } while (0) size_t parse_size(const char *size); @@ -116,6 +114,8 @@ struct kvm_random_state { extern u32 kvm_random_seed; extern struct kvm_random_state kvm_rng; +extern bool kvm_has_gmem_attributes; + struct kvm_random_state new_kvm_random_state(u32 seed); u32 kvm_random_u32(struct kvm_random_state *state); @@ -172,6 +172,7 @@ struct vm_mem_backing_src_alias { bool thp_configured(void); size_t get_trans_hugepagesz(void); size_t get_def_hugetlb_pagesz(void); +size_t get_free_hugepages(size_t page_size); const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(u32 i); size_t get_backing_src_pagesz(u32 i); bool is_backing_src_hugetlb(u32 i); @@ -238,7 +239,6 @@ static inline u32 atoi_non_negative(const char *name, const char *num_str) } int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args); -__printf(3, 4) int guest_snprintf(char *buf, int n, const char *fmt, ...); char *strdup_printf(const char *fmt, ...) __attribute__((format(printf, 1, 2), nonnull(1)));
diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h index 82a8ea6..491fb81 100644 --- a/tools/testing/selftests/kvm/include/x86/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86/evmcs.h
@@ -245,32 +245,18 @@ static inline void evmcs_enable(void) enable_evmcs = true; } -static inline int evmcs_vmptrld(u64 vmcs_pa, void *vmcs) +static inline void evmcs_vmptrld(u64 vmcs_pa, void *vmcs) { current_vp_assist->current_nested_vmcs = vmcs_pa; current_vp_assist->enlighten_vmentry = 1; current_evmcs = vmcs; - - return 0; } -static inline bool load_evmcs(struct hyperv_test_pages *hv) +static inline void load_evmcs(struct hyperv_test_pages *hv) { - if (evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs)) - return false; - + evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs); current_evmcs->revision_id = EVMCS_VERSION; - - return true; -} - -static inline int evmcs_vmptrst(u64 *value) -{ - *value = current_vp_assist->current_nested_vmcs & - ~HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; - - return 0; } static inline int evmcs_vmread(u64 encoding, u64 *value) @@ -1201,62 +1187,4 @@ static inline int evmcs_vmwrite(u64 encoding, u64 value) return 0; } -static inline int evmcs_vmlaunch(void) -{ - int ret; - - current_evmcs->hv_clean_fields = 0; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - -/* - * No guest state (e.g. GPRs) is established by this vmresume. - */ -static inline int evmcs_vmresume(void) -{ - int ret; - - /* HOST_RIP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; - /* HOST_RSP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - #endif /* !SELFTEST_KVM_EVMCS_H */
diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f700..5898a27 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h
@@ -23,6 +23,8 @@ extern bool host_cpu_is_intel; extern bool host_cpu_is_amd; extern bool host_cpu_is_hygon; extern bool host_cpu_is_amd_compatible; +extern bool host_cpu_is_zhaoxin; +extern bool host_cpu_is_intel_compatible; extern u64 guest_tsc_khz; extern struct kvm_mmu guest_mmu; @@ -216,6 +218,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) +#define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) @@ -782,6 +785,12 @@ static inline bool this_cpu_is_hygon(void) return this_cpu_vendor_string_is("HygonGenuine"); } +static inline bool this_cpu_is_zhaoxin(void) +{ + return this_cpu_vendor_string_is("CentaurHauls") || + this_cpu_vendor_string_is(" Shanghai "); +} + static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi) { u32 gprs[4]; @@ -1172,7 +1181,6 @@ static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, struct kvm_x86_cpu_property property, u32 value); -void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, u8 maxphyaddr); void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function); @@ -1318,7 +1326,7 @@ gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages); "lea 1f(%%rip), %%r10\n\t" \ "lea 2f(%%rip), %%r11\n\t" \ fep "1: " insn "\n\t" \ - "xor %%r9, %%r9\n\t" \ + "mov $0, %%r9d\n\t" \ "2:\n\t" \ "mov %%r9b, %[vector]\n\t" \ "mov %%r10, %[error_code]\n\t" @@ -1446,7 +1454,7 @@ static inline bool kvm_is_pmu_enabled(void) static inline bool kvm_is_mediated_pmu_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("enable_mediated_pmu"); return get_kvm_amd_param_bool("enable_mediated_pmu"); @@ -1472,6 +1480,19 @@ static inline bool kvm_is_lbrv_enabled(void) return !!get_kvm_amd_param_integer("lbrv"); } +/* + * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() + * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested + * virtualization support. Use this *only* to validate KVM's own enumeration. + */ +static inline bool kvm_is_nested_virtualization_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("nested"); + + return get_kvm_amd_param_integer("nested"); +} + u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3);
diff --git a/tools/testing/selftests/kvm/include/x86/smm.h b/tools/testing/selftests/kvm/include/x86/smm.h index 2d1afa0..15faaa0 100644 --- a/tools/testing/selftests/kvm/include/x86/smm.h +++ b/tools/testing/selftests/kvm/include/x86/smm.h
@@ -8,7 +8,7 @@ #define SMRAM_MEMSLOT ((1 << 16) | 1) #define SMRAM_PAGES (SMRAM_SIZE / PAGE_SIZE) -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size); void inject_smi(struct kvm_vcpu *vcpu);
diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 04f5e34..8fd9743 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h
@@ -309,125 +309,67 @@ struct vmx_msr_entry { #include "evmcs.h" -static inline int vmxon(u64 phys) -{ - u8 ret; +#define GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, __r, __pa) \ + __GUEST_ASSERT(!__r, __stringify(insn) "[0x%lx] hit %s", \ + __pa, __r < 0 ? "VM-Fail" : ex_str(__r)) - __asm__ __volatile__ ("vmxon %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(phys) - : "cc", "memory"); - - return ret; +#define BUILD_VMCS_ASM_HELPERS(insn) \ +static inline int __##insn(u64 vmcs_pa) \ +{ \ + u64 error_code; \ + u8 vector; \ + u8 failed; \ + \ + asm volatile(KVM_ASM_SAFE(__stringify(insn) " %[pa]") \ + "\n\tsetna %[failed]" \ + : KVM_ASM_SAFE_OUTPUTS(vector, error_code), \ + [failed]"=qm"(failed) \ + : [pa]"m"(vmcs_pa) \ + : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); \ + \ + return vector ? vector : failed ? -EINVAL : 0; \ +} \ + \ +static inline void insn(u64 vmcs_pa) \ +{ \ + int ret = __##insn(vmcs_pa); \ + \ + GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, ret, vmcs_pa); \ } +BUILD_VMCS_ASM_HELPERS(vmxon) +BUILD_VMCS_ASM_HELPERS(vmptrld) +BUILD_VMCS_ASM_HELPERS(vmclear) + static inline void vmxoff(void) { __asm__ __volatile__("vmxoff"); } -static inline int vmclear(u64 vmcs_pa) -{ - u8 ret; - - __asm__ __volatile__ ("vmclear %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - -static inline int vmptrld(u64 vmcs_pa) -{ - u8 ret; - - if (enable_evmcs) - return -1; - - __asm__ __volatile__ ("vmptrld %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - -static inline int vmptrst(u64 *value) -{ - u64 tmp; - u8 ret; - - if (enable_evmcs) - return evmcs_vmptrst(value); - - __asm__ __volatile__("vmptrst %[value]; setna %[ret]" - : [value]"=m"(tmp), [ret]"=rm"(ret) - : : "cc", "memory"); - - *value = tmp; - return ret; -} - -/* - * A wrapper around vmptrst that ignores errors and returns zero if the - * vmptrst instruction fails. - */ -static inline u64 vmptrstz(void) +static inline u64 vmptrst(void) { u64 value = 0; - vmptrst(&value); + u8 ret; + + __asm__ __volatile__("vmptrst %[value]; setna %[ret]" + : [value]"=m"(value), [ret]"=rm"(ret) + : : "cc", "memory"); + + __GUEST_ASSERT(!ret, "vmptrst failed"); return value; } -static inline int vmlaunch(void) +int __vmlaunch(void); +int __vmresume(void); + +static inline void vmlaunch(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmlaunch(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + __GUEST_ASSERT(!__vmlaunch(), "vmlaunch hit VM-Fail"); } -static inline int vmresume(void) +static inline void vmresume(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmresume(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + __GUEST_ASSERT(!__vmresume(), "vmresume hit VM-Fail"); } static inline void vmcall(void) @@ -444,7 +386,7 @@ static inline void vmcall(void) "r10", "r11", "r12", "r13", "r14", "r15"); } -static inline int vmread(u64 encoding, u64 *value) +static inline int __vmread(u64 encoding, u64 *value) { u64 tmp; u8 ret; @@ -457,22 +399,21 @@ static inline int vmread(u64 encoding, u64 *value) : [encoding]"r"(encoding) : "cc", "memory"); - *value = tmp; + if (!ret) + *value = tmp; return ret; } -/* - * A wrapper around vmread that ignores errors and returns zero if the - * vmread instruction fails. - */ -static inline u64 vmreadz(u64 encoding) +static inline u64 vmread(u64 encoding) { u64 value = 0; - vmread(encoding, &value); + + __GUEST_ASSERT(!__vmread(encoding, &value), + "vmread[0x%lx] hit VM-Fail", encoding); return value; } -static inline int vmwrite(u64 encoding, u64 value) +static __always_inline int __vmwrite(u64 encoding, u64 value) { u8 ret; @@ -487,6 +428,12 @@ static inline int vmwrite(u64 encoding, u64 value) return ret; } +static inline void vmwrite(u64 encoding, u64 value) +{ + __GUEST_ASSERT(!__vmwrite(encoding, value), + "vmwrite[0x%lx] = 0x%lx hit VM-Fail", encoding, value); +} + static inline u32 vmcs_revision(void) { return rdmsr(MSR_IA32_VMX_BASIC); @@ -550,13 +497,24 @@ union vmx_ctrl_msr { }; struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva); -bool prepare_for_vmx_operation(struct vmx_pages *vmx); +void prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip); -bool load_vmcs(struct vmx_pages *vmx); +void load_vmcs(struct vmx_pages *vmx); bool ept_1g_pages_supported(void); -bool kvm_cpu_has_ept(void); +bool kvm_cpu_has_secondary_exec_control(u32 ctrl); + +static inline bool kvm_cpu_has_ept(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_ENABLE_EPT); +} + +static inline bool kvm_cpu_has_vmx_virtualize_apic_accesses(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + void vm_enable_ept(struct kvm_vm *vm); void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm);
diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index 168c527..791e576 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c
@@ -6,7 +6,6 @@ #include "proc_util.h" #include <libvfio.h> -#include <linux/sizes.h> #include <stdio.h> #include <stdlib.h> #include <unistd.h> @@ -101,7 +100,6 @@ static int vfio_setup_msi(struct vfio_pci_device *device) allocator = iova_allocator_init(device->iommu); region = &device->driver.region; - region->size = SZ_2M; region->iova = iova_allocator_alloc(allocator, region->size); region->vaddr = kvm_mmap(region->size, prot, flags, -1); TEST_ASSERT(region->vaddr != MAP_FAILED, "mmap() failed\n");
diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c index c531073..22279c63 100644 --- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c +++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
@@ -39,6 +39,12 @@ int main(int argc, char *argv[]) int kvm_max_vcpu_id = kvm_check_cap(KVM_CAP_MAX_VCPU_ID); int kvm_max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); + /* + * PowerPC's PAPR L0 does not guarantee KVM_CAP_MAX_VCPUS when running + * on pseries LPAR (nested kvm-on-powervm). + */ + TEST_REQUIRE(!kvm_is_ppc_pseries_lpar()); + pr_info("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id); pr_info("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus); @@ -58,7 +64,13 @@ int main(int argc, char *argv[]) test_vcpu_creation(0, kvm_max_vcpus); - if (kvm_max_vcpu_id > kvm_max_vcpus) + /* + * Skip the vCPU ID test when running on PowerPC with SMT support, in + * which case system topology is encoded into the vCPU ID and so a VM + * can't use the full advertised vCPU ID range without crafting a valid + * platform specific topology. + */ + if (kvm_max_vcpu_id > kvm_max_vcpus && !kvm_check_cap(KVM_CAP_PPC_SMT)) test_vcpu_creation( kvm_max_vcpu_id - kvm_max_vcpus, kvm_max_vcpus);
diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 01325bf..2dcaa2c 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c
@@ -115,9 +115,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages); vm->mmu.pgd_created = true; }
diff --git a/tools/testing/selftests/kvm/lib/elf.c b/tools/testing/selftests/kvm/lib/elf.c index 1924a98..d5adb68 100644 --- a/tools/testing/selftests/kvm/lib/elf.c +++ b/tools/testing/selftests/kvm/lib/elf.c
@@ -12,6 +12,44 @@ #include "kvm_util.h" +static ssize_t elf_read(int fd, void *buf, size_t count) +{ + ssize_t rc; + ssize_t num_read = 0; + size_t num_left = count; + char *ptr = buf; + + TEST_ASSERT(count, "Count must be non-zero"); + + do { + rc = read(fd, ptr, num_left); + + switch (rc) { + case -1: + TEST_ASSERT(errno == EAGAIN || errno == EINTR, + "Unexpected read failure,\n" + " rc: %zi errno: %i", rc, errno); + break; + + case 0: + TEST_FAIL("Unexpected EOF,\n" + " rc: %zi num_read: %zi num_left: %zu", + rc, num_read, num_left); + break; + + default: + TEST_ASSERT(rc > 0, "Unexpected ret from read,\n" + " rc: %zi errno: %i", rc, errno); + num_read += rc; + num_left -= rc; + ptr += rc; + break; + } + } while (num_read < count); + + return num_read; +} + static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) { off_t offset_rv; @@ -31,7 +69,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) * the real size of the ELF header. */ unsigned char ident[EI_NIDENT]; - test_read(fd, ident, sizeof(ident)); + elf_read(fd, ident, sizeof(ident)); TEST_ASSERT((ident[EI_MAG0] == ELFMAG0) && (ident[EI_MAG1] == ELFMAG1) && (ident[EI_MAG2] == ELFMAG2) && (ident[EI_MAG3] == ELFMAG3), "ELF MAGIC Mismatch,\n" @@ -79,7 +117,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) offset_rv = lseek(fd, 0, SEEK_SET); TEST_ASSERT(offset_rv == 0, "Seek to ELF header failed,\n" " rv: %zi expected: %i", offset_rv, 0); - test_read(fd, hdrp, sizeof(*hdrp)); + elf_read(fd, hdrp, sizeof(*hdrp)); TEST_ASSERT(hdrp->e_phentsize == sizeof(Elf64_Phdr), "Unexpected physical header size,\n" " hdrp->e_phentsize: %x\n" @@ -146,7 +184,7 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) /* Read in the program header. */ Elf64_Phdr phdr; - test_read(fd, &phdr, sizeof(phdr)); + elf_read(fd, &phdr, sizeof(phdr)); /* Skip if this header doesn't describe a loadable segment. */ if (phdr.p_type != PT_LOAD) @@ -186,8 +224,8 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) " expected: 0x%jx", n1, errno, (intmax_t) offset_rv, (intmax_t) phdr.p_offset); - test_read(fd, addr_gva2hva(vm, phdr.p_vaddr), - phdr.p_filesz); + elf_read(fd, addr_gva2hva(vm, phdr.p_vaddr), + phdr.p_filesz); } } close(fd);
diff --git a/tools/testing/selftests/kvm/lib/guest_modes.c b/tools/testing/selftests/kvm/lib/guest_modes.c index 7a96c43..439766f 100644 --- a/tools/testing/selftests/kvm/lib/guest_modes.c +++ b/tools/testing/selftests/kvm/lib/guest_modes.c
@@ -4,16 +4,20 @@ */ #include "guest_modes.h" -#if defined(__aarch64__) || defined(__riscv) +#if defined(__aarch64__) || defined(__riscv) || defined(__powerpc64__) #include "processor.h" enum vm_guest_mode vm_mode_default; #endif +#if defined(__powerpc64__) +#include <unistd.h> +#endif + struct guest_mode guest_modes[NUM_VM_MODES]; void guest_modes_append_default(void) { -#if !defined(__aarch64__) && !defined(__riscv) +#if !defined(__aarch64__) && !defined(__riscv) && !defined(__powerpc64__) guest_mode_append(VM_MODE_DEFAULT, true); #endif @@ -108,6 +112,18 @@ void guest_modes_append_default(void) TEST_ASSERT(vm_mode_default != NUM_VM_MODES, "No supported mode!"); } #endif +#ifdef __powerpc64__ + { + TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX)); + /* Radix guest EA and RA are 52-bit on POWER9 and POWER10 */ + if (sysconf(_SC_PAGESIZE) == 4096) + vm_mode_default = VM_MODE_P52V52_4K; + else + vm_mode_default = VM_MODE_P52V52_64K; + guest_mode_append(VM_MODE_P52V52_4K, true); + guest_mode_append(VM_MODE_P52V52_64K, true); + } +#endif } void for_each_guest_mode(void (*func)(enum vm_guest_mode, void *), void *arg)
diff --git a/tools/testing/selftests/kvm/lib/guest_sprintf.c b/tools/testing/selftests/kvm/lib/guest_sprintf.c index 7a33965..6caebb4 100644 --- a/tools/testing/selftests/kvm/lib/guest_sprintf.c +++ b/tools/testing/selftests/kvm/lib/guest_sprintf.c
@@ -300,15 +300,3 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args) *str = '\0'; return str - buf; } - -int guest_snprintf(char *buf, int n, const char *fmt, ...) -{ - va_list va; - int len; - - va_start(va, fmt); - len = guest_vsnprintf(buf, n, fmt, va); - va_end(va); - - return len; -}
diff --git a/tools/testing/selftests/kvm/lib/io.c b/tools/testing/selftests/kvm/lib/io.c deleted file mode 100644 index fedb2a7..0000000 --- a/tools/testing/selftests/kvm/lib/io.c +++ /dev/null
@@ -1,157 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * tools/testing/selftests/kvm/lib/io.c - * - * Copyright (C) 2018, Google LLC. - */ - -#include "test_util.h" - -/* Test Write - * - * A wrapper for write(2), that automatically handles the following - * special conditions: - * - * + Interrupted system call (EINTR) - * + Write of less than requested amount - * + Non-block return (EAGAIN) - * - * For each of the above, an additional write is performed to automatically - * continue writing the requested data. - * There are also many cases where write(2) can return an unexpected - * error (e.g. EIO). Such errors cause a TEST_ASSERT failure. - * - * Note, for function signature compatibility with write(2), this function - * returns the number of bytes written, but that value will always be equal - * to the number of requested bytes. All other conditions in this and - * future enhancements to this function either automatically issue another - * write(2) or cause a TEST_ASSERT failure. - * - * Args: - * fd - Opened file descriptor to file to be written. - * count - Number of bytes to write. - * - * Output: - * buf - Starting address of data to be written. - * - * Return: - * On success, number of bytes written. - * On failure, a TEST_ASSERT failure is caused. - */ -ssize_t test_write(int fd, const void *buf, size_t count) -{ - ssize_t rc; - ssize_t num_written = 0; - size_t num_left = count; - const char *ptr = buf; - - /* Note: Count of zero is allowed (see "RETURN VALUE" portion of - * write(2) manpage for details. - */ - TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); - - do { - rc = write(fd, ptr, num_left); - - switch (rc) { - case -1: - TEST_ASSERT(errno == EAGAIN || errno == EINTR, - "Unexpected write failure,\n" - " rc: %zi errno: %i", rc, errno); - continue; - - case 0: - TEST_FAIL("Unexpected EOF,\n" - " rc: %zi num_written: %zi num_left: %zu", - rc, num_written, num_left); - break; - - default: - TEST_ASSERT(rc >= 0, "Unexpected ret from write,\n" - " rc: %zi errno: %i", rc, errno); - num_written += rc; - num_left -= rc; - ptr += rc; - break; - } - } while (num_written < count); - - return num_written; -} - -/* Test Read - * - * A wrapper for read(2), that automatically handles the following - * special conditions: - * - * + Interrupted system call (EINTR) - * + Read of less than requested amount - * + Non-block return (EAGAIN) - * - * For each of the above, an additional read is performed to automatically - * continue reading the requested data. - * There are also many cases where read(2) can return an unexpected - * error (e.g. EIO). Such errors cause a TEST_ASSERT failure. Note, - * it is expected that the file opened by fd at the current file position - * contains at least the number of requested bytes to be read. A TEST_ASSERT - * failure is produced if an End-Of-File condition occurs, before all the - * data is read. It is the callers responsibility to assure that sufficient - * data exists. - * - * Note, for function signature compatibility with read(2), this function - * returns the number of bytes read, but that value will always be equal - * to the number of requested bytes. All other conditions in this and - * future enhancements to this function either automatically issue another - * read(2) or cause a TEST_ASSERT failure. - * - * Args: - * fd - Opened file descriptor to file to be read. - * count - Number of bytes to read. - * - * Output: - * buf - Starting address of where to write the bytes read. - * - * Return: - * On success, number of bytes read. - * On failure, a TEST_ASSERT failure is caused. - */ -ssize_t test_read(int fd, void *buf, size_t count) -{ - ssize_t rc; - ssize_t num_read = 0; - size_t num_left = count; - char *ptr = buf; - - /* Note: Count of zero is allowed (see "If count is zero" portion of - * read(2) manpage for details. - */ - TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); - - do { - rc = read(fd, ptr, num_left); - - switch (rc) { - case -1: - TEST_ASSERT(errno == EAGAIN || errno == EINTR, - "Unexpected read failure,\n" - " rc: %zi errno: %i", rc, errno); - break; - - case 0: - TEST_FAIL("Unexpected EOF,\n" - " rc: %zi num_read: %zi num_left: %zu", - rc, num_read, num_left); - break; - - default: - TEST_ASSERT(rc > 0, "Unexpected ret from read,\n" - " rc: %zi errno: %i", rc, errno); - num_read += rc; - num_left -= rc; - ptr += rc; - break; - } - } while (num_read < count); - - return num_read; -}
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9ddc047..d42e201 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -17,8 +17,11 @@ #include <sys/stat.h> #include <time.h> #include <unistd.h> +#include <linux/align.h> #include <linux/kernel.h> +#include "../../../../../mm/gup_test.h" + #define KVM_UTIL_MIN_PFN 2 u32 kvm_random_seed; @@ -32,6 +35,8 @@ static void kvm_seed_rng(u32 seed) kvm_rng = new_kvm_random_state(kvm_random_seed); } +bool kvm_has_gmem_attributes; + static size_t vcpu_mmap_sz(void); int __open_path_or_exit(const char *path, int flags, const char *enoent_help) @@ -226,6 +231,8 @@ const char *vm_guest_mode_string(u32 i) [VM_MODE_P41V57_4K] = "PA-bits:41, VA-bits:57, 4K pages", [VM_MODE_P41V48_4K] = "PA-bits:41, VA-bits:48, 4K pages", [VM_MODE_P41V39_4K] = "PA-bits:41, VA-bits:39, 4K pages", + [VM_MODE_P52V52_4K] = "PA-bits:52, VA-bits:52, 4K pages", + [VM_MODE_P52V52_64K] = "PA-bits:52, VA-bits:52, 64K pages", }; _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES, "Missing new mode strings?"); @@ -262,6 +269,8 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { [VM_MODE_P41V57_4K] = { 41, 57, 0x1000, 12 }, [VM_MODE_P41V48_4K] = { 41, 48, 0x1000, 12 }, [VM_MODE_P41V39_4K] = { 41, 39, 0x1000, 12 }, + [VM_MODE_P52V52_4K] = { 52, 52, 0x1000, 12 }, + [VM_MODE_P52V52_64K] = { 52, 52, 0x10000, 16 }, }; _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES, "Missing new mode params?"); @@ -287,6 +296,7 @@ __weak void vm_populate_gva_bitmap(struct kvm_vm *vm) struct kvm_vm *____vm_create(struct vm_shape shape) { struct kvm_vm *vm; + int i; vm = calloc(1, sizeof(*vm)); TEST_ASSERT(vm != NULL, "Insufficient Memory"); @@ -295,6 +305,8 @@ struct kvm_vm *____vm_create(struct vm_shape shape) vm->regions.gpa_tree = RB_ROOT; vm->regions.hva_tree = RB_ROOT; hash_init(vm->regions.slot_hash); + for (i = 0; i < NR_MEM_REGIONS; i++) + vm->memslots[i] = KVM_INVALID_MEMSLOT; vm->mode = shape.mode; vm->type = shape.type; @@ -379,6 +391,10 @@ struct kvm_vm *____vm_create(struct vm_shape shape) case VM_MODE_P41V39_4K: vm->mmu.pgtable_levels = 3; break; + case VM_MODE_P52V52_4K: + case VM_MODE_P52V52_64K: + vm->mmu.pgtable_levels = 4; + break; default: TEST_FAIL("Unknown guest mode: 0x%x", vm->mode); } @@ -490,8 +506,9 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, u64 nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; + u64 gmem_flags = 0; struct kvm_vm *vm; - int i, flags; + int flags; kvm_set_files_rlimit(nr_runnable_vcpus); @@ -505,12 +522,24 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, * for CoCo VMs that require GUEST_MEMFD backed private memory. */ flags = 0; - if (is_guest_memfd_required(shape)) + if (is_guest_memfd_required(shape)) { flags |= KVM_MEM_GUEST_MEMFD; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags); - for (i = 0; i < NR_MEM_REGIONS; i++) - vm->memslots[i] = 0; + /* + * Allow mmap() and initialize the gmem instance as SHARED if + * in-place conversions are supported, so that the memory can + * be populated/initialized like any other memory region. + */ + if (kvm_has_gmem_attributes) + gmem_flags |= GUEST_MEMFD_FLAG_INIT_SHARED | + GUEST_MEMFD_FLAG_MMAP; + } + + vm_mem_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags, -1, 0, gmem_flags); + ____vm_override_mem_region(vm, MEM_REGION_CODE, 0); + ____vm_override_mem_region(vm, MEM_REGION_PT, 0); + ____vm_override_mem_region(vm, MEM_REGION_DATA, 0); + ____vm_override_mem_region(vm, MEM_REGION_TEST_DATA, 0); kvm_vm_elf_load(vm, program_invocation_name); @@ -527,6 +556,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, kvm_seed_rng(kvm_random_seed); sync_global_to_guest(vm, kvm_rng); + sync_global_to_guest(vm, kvm_has_gmem_attributes); kvm_arch_vm_post_create(vm, nr_runnable_vcpus); @@ -642,6 +672,29 @@ int __pin_task_to_cpu(pthread_t task, int cpu) return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset); } +static int gup_test_fd = -1; + +void pin_pages(void *vaddr, uint64_t size) +{ + const struct pin_longterm_test args = { + .addr = (uint64_t)vaddr, + .size = size, + .flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE, + }; + + gup_test_fd = __open_path_or_exit("/sys/kernel/debug/gup_test", O_RDWR, + "Is CONFIG_GUP_TEST enabled?"); + + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0); +} + +void unpin_pages(void) +{ + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0); + + kvm_free_fd(gup_test_fd); +} + static u32 parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask) { u32 pcpu = atoi_non_negative("CPU number", cpu_str); @@ -970,7 +1023,7 @@ void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { struct kvm_userspace_memory_region2 region = { .slot = slot, @@ -978,8 +1031,8 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, .guest_phys_addr = gpa, .memory_size = size, .userspace_addr = (uintptr_t)hva, - .guest_memfd = guest_memfd, - .guest_memfd_offset = guest_memfd_offset, + .guest_memfd = gmem_fd, + .guest_memfd_offset = gmem_offset, }; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -989,10 +1042,10 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva, - guest_memfd, guest_memfd_offset); + gmem_fd, gmem_offset); TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)", errno, strerror(errno)); @@ -1002,13 +1055,16 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd, u64 guest_memfd_offset) + int gmem_fd, u64 gmem_offset, u64 gmem_flags) { int ret; struct userspace_mem_region *region; - size_t backing_src_pagesz = get_backing_src_pagesz(src_type); size_t mem_size = npages * vm->page_size; - size_t alignment = 1; + size_t backing_src_pagesz; + off_t mmap_offset; + bool is_gmem_mmap; + size_t alignment; + int mmap_flags; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -1060,58 +1116,40 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* Allocate and initialize new mem region structure. */ region = calloc(1, sizeof(*region)); TEST_ASSERT(region != NULL, "Insufficient Memory"); - region->mmap_size = mem_size; - /* - * When using THP mmap is not guaranteed to returned a hugepage aligned - * address so we have to pad the mmap. Padding is not needed for HugeTLB - * because mmap will always return an address aligned to the HugeTLB - * page size. - */ - if (src_type == VM_MEM_SRC_ANONYMOUS_THP) - alignment = max(backing_src_pagesz, alignment); + is_gmem_mmap = (flags & KVM_MEM_GUEST_MEMFD) && + (gmem_flags & GUEST_MEMFD_FLAG_MMAP); + + if (is_gmem_mmap) { + backing_src_pagesz = getpagesize(); + alignment = 1; + mmap_flags = MAP_SHARED; + mmap_offset = gmem_offset; + } else { + backing_src_pagesz = get_backing_src_pagesz(src_type); + /* + * When using THP mmap is not guaranteed to returned a hugepage aligned + * address so we have to pad the mmap. Padding is not needed for HugeTLB + * because mmap will always return an address aligned to the HugeTLB + * page size. + */ + alignment = src_type == VM_MEM_SRC_ANONYMOUS_THP ? backing_src_pagesz : 1; + mmap_flags = vm_mem_backing_src_alias(src_type)->flag; + mmap_offset = 0; + } TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz)); + region->mmap_size = mem_size; /* Add enough memory to align up if necessary */ if (alignment > 1) region->mmap_size += alignment; - region->fd = -1; - if (backing_src_is_shared(src_type)) - region->fd = kvm_memfd_alloc(region->mmap_size, - src_type == VM_MEM_SRC_SHARED_HUGETLB); - - region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); - - TEST_ASSERT(!is_backing_src_hugetlb(src_type) || - region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz), - "mmap_start %p is not aligned to HugeTLB page size 0x%lx", - region->mmap_start, backing_src_pagesz); - - /* Align host address */ - region->host_mem = align_ptr_up(region->mmap_start, alignment); - - /* As needed perform madvise */ - if ((src_type == VM_MEM_SRC_ANONYMOUS || - src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { - ret = madvise(region->host_mem, mem_size, - src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); - TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", - region->host_mem, mem_size, - vm_mem_backing_src_alias(src_type)->name); - } - - region->backing_src_type = src_type; - if (flags & KVM_MEM_GUEST_MEMFD) { - if (guest_memfd < 0) { - u32 guest_memfd_flags = 0; - TEST_ASSERT(!guest_memfd_offset, + if (gmem_fd < 0) { + TEST_ASSERT(!gmem_offset, "Offset must be zero when creating new guest_memfd"); - guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); + gmem_fd = vm_create_guest_memfd(vm, mem_size, gmem_flags); } else { /* * Install a unique fd for each memslot so that the fd @@ -1119,15 +1157,51 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, * needing to track if the fd is owned by the framework * or by the caller. */ - guest_memfd = kvm_dup(guest_memfd); + gmem_fd = kvm_dup(gmem_fd); } - region->region.guest_memfd = guest_memfd; - region->region.guest_memfd_offset = guest_memfd_offset; + region->region.guest_memfd = gmem_fd; + region->region.guest_memfd_offset = gmem_offset; } else { region->region.guest_memfd = -1; } + region->fd = -1; + if (is_gmem_mmap) { + region->fd = kvm_dup(gmem_fd); + } else if (backing_src_is_shared(src_type)) { + region->fd = kvm_memfd_alloc(region->mmap_size, + src_type == VM_MEM_SRC_SHARED_HUGETLB); + } + + region->mmap_start = __kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, + mmap_flags, region->fd, mmap_offset); + + /* Align host address */ + region->host_mem = align_ptr_up(region->mmap_start, alignment); + + if (!is_gmem_mmap) { + TEST_ASSERT(!is_backing_src_hugetlb(src_type) || + region->mmap_start == + align_ptr_up(region->mmap_start, backing_src_pagesz), + "mmap_start %p is not aligned to HugeTLB page size 0x%lx", + region->mmap_start, backing_src_pagesz); + + /* As needed perform madvise */ + if ((src_type == VM_MEM_SRC_ANONYMOUS || + src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { + int advice = src_type == VM_MEM_SRC_ANONYMOUS ? + MADV_NOHUGEPAGE : MADV_HUGEPAGE; + + ret = madvise(region->host_mem, mem_size, advice); + TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", + region->host_mem, mem_size, + vm_mem_backing_src_alias(src_type)->name); + } + } + + region->backing_src_type = src_type; + region->unused_phy_pages = sparsebit_alloc(); if (vm_arch_has_protected_memory(vm)) region->protected_phy_pages = sparsebit_alloc(); @@ -1152,10 +1226,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* If shared memory, create an alias. */ if (region->fd >= 0) { - region->mmap_alias = kvm_mmap(region->mmap_size, - PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); + region->mmap_alias = __kvm_mmap(region->mmap_size, + PROT_READ | PROT_WRITE, + mmap_flags, region->fd, + mmap_offset); /* Align host alias address */ region->host_alias = align_ptr_up(region->mmap_alias, alignment); @@ -1166,7 +1240,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags) { - vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0); + vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0, 0); } /* @@ -1189,6 +1263,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot) { struct userspace_mem_region *region; + TEST_ASSERT(memslot != KVM_INVALID_MEMSLOT, "vm->memslots[] unpopulated?"); + hash_for_each_possible(vm->regions.slot_hash, region, slot_node, memslot) if (region->region.slot == memslot) @@ -1299,27 +1375,20 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, u64 base, u64 size, bool punch_hole) { const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0); - struct userspace_mem_region *region; u64 end = base + size; - gpa_t gpa, len; off_t fd_offset; - int ret; + int fd, ret; + size_t len; + gpa_t gpa; for (gpa = base; gpa < end; gpa += len) { - u64 offset; + fd = kvm_gpa_to_guest_memfd(vm, gpa, &fd_offset, &len); + len = min(end - gpa, len); - region = userspace_mem_region_find(vm, gpa, gpa); - TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, - "Private memory region not found for GPA 0x%lx", gpa); - - offset = gpa - region->region.guest_phys_addr; - fd_offset = region->region.guest_memfd_offset + offset; - len = min_t(u64, end - gpa, region->region.memory_size - offset); - - ret = fallocate(region->region.guest_memfd, mode, fd_offset, len); + ret = fallocate(fd, mode, fd_offset, len); TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx", punch_hole ? "punch hole" : "allocate", gpa, len, - region->region.guest_memfd, mode, fd_offset); + fd, mode, fd_offset); } } @@ -1465,9 +1534,7 @@ static gva_t ____vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva, u64 pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0); virt_pgd_alloc(vm); - gpa_t gpa = __vm_phy_pages_alloc(vm, pages, - KVM_UTIL_MIN_PFN * vm->page_size, - vm->memslots[type], protected); + gpa_t gpa = __vm_phy_pages_alloc(vm, pages, type, protected); /* * Find an unused range of virtual page addresses of at least @@ -1656,6 +1723,22 @@ void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa) return (void *) ((uintptr_t) region->host_alias + offset); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes) +{ + struct userspace_mem_region *region; + gpa_t gpa_offset; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, + "guest_memfd memory region not found for GPA 0x%lx", gpa); + + gpa_offset = gpa - region->region.guest_phys_addr; + *fd_offset = region->region.guest_memfd_offset + gpa_offset; + *nr_bytes = region->region.memory_size - gpa_offset; + return region->region.guest_memfd; +} + /* Create an interrupt controller chip for the specified VM. */ void vm_create_irqchip(struct kvm_vm *vm) { @@ -1727,6 +1810,7 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu) TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0"); reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64)); + TEST_ASSERT(reg_list, "Failed to allocate reg_list"); reg_list->n = reg_list_n.n; vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list); return reg_list; @@ -2025,33 +2109,21 @@ const char *exit_reason_str(unsigned int exit_reason) } /* - * Physical Contiguous Page Allocator + * Allocate contiguous (guest) physical pages in a given memory region, at or + * above the minimum specified GPA. If the memory is protected/private, also + * add the allocated pages to the region's set of protected pages, e.g. so that + * arch code knows which pages need to be encrypted when launching the VM. * - * Input Args: - * vm - Virtual Machine - * num - number of pages - * min_gpa - Physical address minimum - * memslot - Memory region to allocate page from - * protected - True if the pages will be used as protected/private memory - * - * Output Args: None - * - * Return: - * Starting physical address - * - * Within the VM specified by vm, locates a range of available physical - * pages at or above min_gpa. If found, the pages are marked as in use - * and their base address is returned. A TEST_ASSERT failure occurs if - * not enough pages are available at or above min_gpa. + * Note, success is guaranteed! */ -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, - gpa_t min_gpa, u32 memslot, - bool protected) +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned) { + size_t alignment = naturally_aligned ? nr_pages : 1; struct userspace_mem_region *region; sparsebit_idx_t pg, base; - TEST_ASSERT(num > 0, "Must allocate at least one page"); + TEST_ASSERT(nr_pages, "Must allocate at least one page"); TEST_ASSERT((min_gpa % vm->page_size) == 0, "Min physical address " "not divisible by page size.\n" @@ -2062,43 +2134,78 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, TEST_ASSERT(!protected || region->protected_phy_pages, "Region doesn't support protected memory"); - base = pg = min_gpa >> vm->page_shift; - do { - for (; pg < base + num; ++pg) { - if (!sparsebit_is_set(region->unused_phy_pages, pg)) { - base = pg = sparsebit_next_set(region->unused_phy_pages, pg); - break; - } + base = min_gpa >> vm->page_shift; +again: + base = ALIGN(base, alignment); + for (pg = base; pg < base + nr_pages; ++pg) { + if (!sparsebit_is_set(region->unused_phy_pages, pg)) { + base = sparsebit_next_set(region->unused_phy_pages, pg); + if (!base) + goto enomem; + goto again; } - } while (pg && pg != base + num); - - if (pg == 0) { - fprintf(stderr, "No guest physical page available, " - "min_gpa: 0x%lx page_size: 0x%x memslot: %u\n", - min_gpa, vm->page_size, memslot); - fputs("---- vm dump ----\n", stderr); - vm_dump(stderr, vm, 2); - abort(); } - for (pg = base; pg < base + num; ++pg) { + for (pg = base; pg < base + nr_pages; ++pg) { sparsebit_clear(region->unused_phy_pages, pg); if (protected) sparsebit_set(region->protected_phy_pages, pg); } return base * vm->page_size; + +enomem: + fprintf(stderr, "No guest physical page available, min_gpa: 0x%lx page_size: 0x%x memslot: %u\n", + min_gpa, vm->page_size, memslot); + fputs("---- vm dump ----\n", stderr); + vm_dump(stderr, vm, 2); + abort(); + __builtin_unreachable(); } -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot) +__weak bool kvm_arch_needs_naturally_aligned_page_tables(void) { - return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); + return false; } -gpa_t vm_alloc_page_table(struct kvm_vm *vm) +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type, bool protected) { - return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + struct userspace_mem_region *region = vm_get_mem_region(vm, type); + bool naturally_aligned = false; + gpa_t min_gpa; + + TEST_ASSERT(region, "No region for type '%u', memslot '%u'", + type, vm->memslots[type]); + + switch (type) { + case MEM_REGION_CODE: + case MEM_REGION_DATA: + case MEM_REGION_TEST_DATA: + /* + * If the region is backed by the default memslot (id=0), use + * selftests' hardcoded minimum PFN, otherwise use the base of + * the custom memory slot that backs the region. + */ + if (!vm->memslots[type]) + min_gpa = KVM_UTIL_MIN_PFN * vm->page_size; + else + min_gpa = region->region.guest_phys_addr; + break; + case MEM_REGION_PT: + min_gpa = KVM_GUEST_PAGE_TABLE_MIN_PADDR; + naturally_aligned = kvm_arch_needs_naturally_aligned_page_tables(); + break; + case MEM_REGION_TEST_EXTRA: + min_gpa = region->region.guest_phys_addr; + break; + default: + TEST_FAIL("Invalid memory region type '%u'", type); + break; + } + + return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, vm->memslots[type], + protected, naturally_aligned); } /* @@ -2274,13 +2381,20 @@ __weak void kvm_selftest_arch_init(void) { } -static void report_unexpected_signal(int signum) +__thread sigjmp_buf expect_sigbus_jmpbuf; +__thread volatile sig_atomic_t expecting_sigbus; + +void catchall_signal_handler(int signum) { + switch (signum) { + case SIGBUS: { + if (expecting_sigbus) + siglongjmp(expect_sigbus_jmpbuf, 1); + + TEST_FAIL("Unexpected SIGBUS (%d)\n", signum); + } #define KVM_CASE_SIGNUM(sig) \ case sig: TEST_FAIL("Unexpected " #sig " (%d)\n", signum) - - switch (signum) { - KVM_CASE_SIGNUM(SIGBUS); KVM_CASE_SIGNUM(SIGSEGV); KVM_CASE_SIGNUM(SIGILL); KVM_CASE_SIGNUM(SIGFPE); @@ -2292,12 +2406,13 @@ static void report_unexpected_signal(int signum) void __attribute((constructor)) kvm_selftest_init(void) { struct sigaction sig_sa = { - .sa_handler = report_unexpected_signal, + .sa_handler = catchall_signal_handler, }; /* Tell stdout not to buffer its content. */ setbuf(stdout, NULL); + expecting_sigbus = false; sigaction(SIGBUS, &sig_sa, NULL); sigaction(SIGSEGV, &sig_sa, NULL); sigaction(SIGILL, &sig_sa, NULL); @@ -2306,6 +2421,8 @@ void __attribute((constructor)) kvm_selftest_init(void) srandom(time(0)); kvm_seed_rng(random()); + kvm_has_gmem_attributes = kvm_has_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES); + kvm_selftest_arch_init(); }
diff --git a/tools/testing/selftests/kvm/lib/loongarch/processor.c b/tools/testing/selftests/kvm/lib/loongarch/processor.c index 47bbde3..83e53ff 100644 --- a/tools/testing/selftests/kvm/lib/loongarch/processor.c +++ b/tools/testing/selftests/kvm/lib/loongarch/processor.c
@@ -9,7 +9,6 @@ #include "processor.h" #include "ucall_common.h" -#define LOONGARCH_PAGE_TABLE_PHYS_MIN 0x200000 #define LOONGARCH_GUEST_STACK_VADDR_MIN 0x200000 static gpa_t invalid_pgtable[4]; @@ -57,9 +56,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) child = table = 0; for (i = 0; i < vm->mmu.pgtable_levels; i++) { invalid_pgtable[i] = child; - table = vm_phy_page_alloc(vm, LOONGARCH_PAGE_TABLE_PHYS_MIN, - vm->memslots[MEM_REGION_PT]); - TEST_ASSERT(table, "Fail to allocate page tale at level %d\n", i); + table = vm_alloc_page_table(vm); virt_set_pgtable(vm, table, child); child = table; } @@ -278,6 +275,7 @@ static void loongarch_set_csr(struct kvm_vcpu *vcpu, u64 id, u64 val) void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) { + bool has_ptw; int ret, width; unsigned int cfg; unsigned long val; @@ -300,6 +298,12 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_LASX); if (!ret) cfg |= CPUCFG2_LASX; + ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_PTW); + if (!ret) + cfg |= CPUCFG2_PTW; + + has_ptw = !ret; + loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG2, cfg); cfg = read_cpucfg(LOONGARCH_CPUCFG6); loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG6, cfg); @@ -338,6 +342,9 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) /* PGD page shift and width */ val = (vm->page_shift + width * (vm->mmu.pgtable_levels - 1)) | width << 6; + if (has_ptw) + val |= CSR_PWCTL1_PTW; + loongarch_set_csr(vcpu, LOONGARCH_CSR_PWCTL1, val); loongarch_set_csr(vcpu, LOONGARCH_CSR_PGDL, vm->mmu.pgd); @@ -345,8 +352,10 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) * Refill exception runs on real mode * Entry address should be physical address */ - val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); - loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); + if (!has_ptw) { + val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); + loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); + } /* * General exception runs on page-enabled mode
diff --git a/tools/testing/selftests/kvm/lib/powerpc/handlers.S b/tools/testing/selftests/kvm/lib/powerpc/handlers.S new file mode 100644 index 0000000..b860f6a --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/handlers.S
@@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <ppc_asm.h> + +.macro INTERRUPT vec +. = __interrupts_start + \vec + std %r0,(0*8)(%r13) + std %r3,(3*8)(%r13) + mfspr %r0,SPR_CFAR + li %r3,\vec + b handle_interrupt +.endm + +.balign 0x1000 +.global __interrupts_start +__interrupts_start: +INTERRUPT 0x100 +INTERRUPT 0x200 +INTERRUPT 0x300 +INTERRUPT 0x380 +INTERRUPT 0x400 +INTERRUPT 0x480 +INTERRUPT 0x500 +INTERRUPT 0x600 +INTERRUPT 0x700 +INTERRUPT 0x800 +INTERRUPT 0x900 +INTERRUPT 0xa00 +INTERRUPT 0xc00 +INTERRUPT 0xd00 +INTERRUPT 0xf00 +INTERRUPT 0xf20 +INTERRUPT 0xf40 +INTERRUPT 0xf60 + +virt_handle_interrupt: + stdu %r1,-INT_FRAME_SIZE(%r1) + mr %r3,%r31 + bl route_interrupt + ld %r4,(32*8)(%r31) /* NIA */ + ld %r5,(33*8)(%r31) /* MSR */ + ld %r6,(35*8)(%r31) /* LR */ + ld %r7,(36*8)(%r31) /* CTR */ + ld %r8,(37*8)(%r31) /* XER */ + lwz %r9,(38*8)(%r31) /* CR */ + mtspr SPR_SRR0,%r4 + mtspr SPR_SRR1,%r5 + mtlr %r6 + mtctr %r7 + mtxer %r8 + mtcr %r9 +reg=4 + ld %r0,(0*8)(%r31) + ld %r3,(3*8)(%r31) +.rept 28 + ld reg,(reg*8)(%r31) + reg=reg+1 +.endr + addi %r1,%r1,INT_FRAME_SIZE + rfid + +virt_handle_interrupt_p: + .llong virt_handle_interrupt + +handle_interrupt: +reg=4 +.rept 28 + std reg,(reg*8)(%r13) + reg=reg+1 +.endr + mfspr %r4,SPR_SRR0 + mfspr %r5,SPR_SRR1 + mflr %r6 + mfctr %r7 + mfxer %r8 + mfcr %r9 + std %r4,(32*8)(%r13) /* NIA */ + std %r5,(33*8)(%r13) /* MSR */ + std %r0,(34*8)(%r13) /* CFAR */ + std %r6,(35*8)(%r13) /* LR */ + std %r7,(36*8)(%r13) /* CTR */ + std %r8,(37*8)(%r13) /* XER */ + stw %r9,(38*8 + 0)(%r13) /* CR */ + stw %r3,(38*8 + 4)(%r13) /* TRAP */ + + ld %r31,(39*8)(%r13) /* vaddr */ + ld %r4,virt_handle_interrupt_p - __interrupts_start(0) + mtspr SPR_SRR0,%r4 + /* Reuse SRR1 */ + + rfid +.global __interrupts_end +__interrupts_end:
diff --git a/tools/testing/selftests/kvm/lib/powerpc/hcall.c b/tools/testing/selftests/kvm/lib/powerpc/hcall.c new file mode 100644 index 0000000..d5f5282 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/hcall.c
@@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PAPR (pseries) hcall support. + */ +#include "kvm_util.h" +#include "hcall.h" + +s64 hcall0(u64 token) +{ + register uintptr_t r3 asm ("r3") = token; + + asm volatile("sc 1" : "+r"(r3) : + : "r0", "r4", "r5", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} + +s64 hcall1(u64 token, u64 arg1) +{ + register uintptr_t r3 asm ("r3") = token; + register uintptr_t r4 asm ("r4") = arg1; + + asm volatile("sc 1" : "+r"(r3), "+r"(r4) : + : "r0", "r5", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} + +s64 hcall2(u64 token, u64 arg1, u64 arg2) +{ + register uintptr_t r3 asm ("r3") = token; + register uintptr_t r4 asm ("r4") = arg1; + register uintptr_t r5 asm ("r5") = arg2; + + asm volatile("sc 1" : "+r"(r3), "+r"(r4), "+r"(r5) : + : "r0", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +}
diff --git a/tools/testing/selftests/kvm/lib/powerpc/processor.c b/tools/testing/selftests/kvm/lib/powerpc/processor.c new file mode 100644 index 0000000..3014550 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/processor.c
@@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM selftest powerpc library code - CPU-related functions (page tables...) + */ + +#include <linux/sizes.h> + +#include "processor.h" +#include "kvm_util.h" +#include "ucall_common.h" +#include "guest_modes.h" +#include "hcall.h" + +#define RADIX_TREE_SIZE ((0x2UL << 61) | (0x5UL << 5)) /* 52-bits */ +#define RADIX_PGD_INDEX_SIZE 13 + +/* + * A radix table must be aligned to its size, and the tables are larger than + * the guest base page size for several level/page size combinations. + */ +bool kvm_arch_needs_naturally_aligned_page_tables(void) +{ + return true; +} + +static void set_proc_table(struct kvm_vm *vm, int pid, u64 dw0, u64 dw1) +{ + u64 *proc_table; + + proc_table = addr_gpa2hva(vm, vm->arch.prtb); + proc_table[pid * 2 + 0] = cpu_to_be64(dw0); + proc_table[pid * 2 + 1] = cpu_to_be64(dw1); +} + +static void set_radix_proc_table(struct kvm_vm *vm, int pid, gpa_t pgd) +{ + set_proc_table(vm, pid, pgd | RADIX_TREE_SIZE | RADIX_PGD_INDEX_SIZE, 0); +} + +void virt_arch_pgd_alloc(struct kvm_vm *vm) +{ + struct kvm_ppc_mmuv3_cfg mmu_cfg; + gpa_t prtb, pgtb; + size_t pgd_pages; + + TEST_ASSERT((vm->mode == VM_MODE_P52V52_4K) || + (vm->mode == VM_MODE_P52V52_64K), + "Unsupported guest mode, mode: 0x%x", vm->mode); + + if (vm->mmu.pgd_created) + return; + + prtb = vm_alloc_page_table(vm); + vm->arch.prtb = prtb; + + pgd_pages = (1UL << (RADIX_PGD_INDEX_SIZE + 3)) >> vm->page_shift; + if (!pgd_pages) + pgd_pages = 1; + pgtb = vm_alloc_page_table_pages(vm, pgd_pages); + vm->mmu.pgd = pgtb; + + /* Set the base page directory in the proc table */ + set_radix_proc_table(vm, 0, pgtb); + + if (vm->mode == VM_MODE_P52V52_4K) + mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x0; /* 4K size */ + else /* vm->mode == VM_MODE_P52V52_64K */ + mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x4; /* 64K size */ + mmu_cfg.flags = KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE; + + vm_ioctl(vm, KVM_PPC_CONFIGURE_V3_MMU, &mmu_cfg); + + vm->mmu.pgd_created = true; +} + +static int pt_shift(struct kvm_vm *vm, int level) +{ + switch (level) { + case 1: + return 13; + case 2: + case 3: + return 9; + case 4: + if (vm->mode == VM_MODE_P52V52_4K) + return 9; + else /* vm->mode == VM_MODE_P52V52_64K */ + return 5; + default: + TEST_ASSERT(false, "Invalid page table level %d\n", level); + return 0; + } +} + +static u64 pt_entry_coverage(struct kvm_vm *vm, int level) +{ + u64 size = vm->page_size; + + if (level == 4) + return size; + size <<= pt_shift(vm, 4); + if (level == 3) + return size; + size <<= pt_shift(vm, 3); + if (level == 2) + return size; + size <<= pt_shift(vm, 2); + return size; +} + +static int pt_idx(struct kvm_vm *vm, gva_t gva, int level, u64 *nls) +{ + switch (level) { + case 1: + if (nls) + *nls = 0x9; + return (gva >> 39) & 0x1fff; + case 2: + if (nls) + *nls = 0x9; + return (gva >> 30) & 0x1ff; + case 3: + if (vm->mode == VM_MODE_P52V52_4K) { + if (nls) + *nls = 0x9; + } else { /* vm->mode == VM_MODE_P52V52_64K */ + if (nls) + *nls = 0x5; + } + return (gva >> 21) & 0x1ff; + case 4: + if (vm->mode == VM_MODE_P52V52_4K) + return (gva >> 12) & 0x1ff; + else /* vm->mode == VM_MODE_P52V52_64K */ + return (gva >> 16) & 0x1f; + default: + TEST_ASSERT(false, "Invalid page table level %d\n", level); + return 0; + } +} + +static u64 *virt_get_pte(struct kvm_vm *vm, gpa_t pt, + gva_t gva, int level, u64 *nls) +{ + int idx = pt_idx(vm, gva, level, nls); + u64 *ptep = addr_gpa2hva(vm, pt + idx * 8); + + return ptep; +} + +#define PTE_VALID 0x8000000000000000ull +#define PTE_LEAF 0x4000000000000000ull +#define PTE_REFERENCED 0x0000000000000100ull +#define PTE_CHANGED 0x0000000000000080ull +#define PTE_PRIV 0x0000000000000008ull +#define PTE_READ 0x0000000000000004ull +#define PTE_RW 0x0000000000000002ull +#define PTE_EXEC 0x0000000000000001ull +#define PTE_PAGE_MASK 0x01fffffffffff000ull + +#define PDE_VALID PTE_VALID +#define PDE_NLS 0x0000000000000011ull +#define PDE_PT_MASK 0x0fffffffffffff00ull + +static gpa_t __vm_alloc_pt(struct kvm_vm *vm, u64 pt_shift) +{ + gpa_t pt; + + if (pt_shift >= vm->page_shift) { + size_t pt_pages = 1ULL << (pt_shift - vm->page_shift); + + pt = vm_alloc_page_table_pages(vm, pt_pages); + } else { + struct vm_pt_frag_cache *pt_frag_cache; + + if (pt_shift == 8) { + pt_frag_cache = &vm->arch.pt_frag_cache[0]; + } else if (pt_shift == 12) { + pt_frag_cache = &vm->arch.pt_frag_cache[1]; + } else { + TEST_ASSERT(0, "Invalid pt_shift:%lu\n", pt_shift); + return 0; + } + + if (!pt_frag_cache->page) + pt_frag_cache->page = vm_alloc_page_table(vm); + pt = pt_frag_cache->page + pt_frag_cache->page_nr_used; + pt_frag_cache->page_nr_used += (1 << pt_shift); + if (pt_frag_cache->page_nr_used == vm->page_size) { + pt_frag_cache->page = 0; + pt_frag_cache->page_nr_used = 0; + } + } + + return pt; +} + +void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa) +{ + gpa_t pt = vm->mmu.pgd; + u64 *ptep, pte; + int level; + + for (level = 1; level <= 3; level++) { + u64 nls; + u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); + u64 pde = be64_to_cpu(*pdep); + + if (pde) { + TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), + "Invalid PDE at level: %u gva: 0x%lx pde:0x%lx\n", + level, gva, pde); + pt = pde & PDE_PT_MASK; + continue; + } + + pt = __vm_alloc_pt(vm, nls + 3); + pde = PDE_VALID | nls | pt; + *pdep = cpu_to_be64(pde); + } + + ptep = virt_get_pte(vm, pt, gva, level, NULL); + pte = be64_to_cpu(*ptep); + + TEST_ASSERT(!pte, "PTE already present at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + pte = PTE_VALID | PTE_LEAF | PTE_REFERENCED | PTE_CHANGED | PTE_PRIV | + PTE_READ | PTE_RW | PTE_EXEC | (gpa & PTE_PAGE_MASK); + *ptep = cpu_to_be64(pte); +} + +gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva) +{ + gpa_t pt = vm->mmu.pgd; + u64 *ptep, pte; + int level; + + for (level = 1; level <= 3; level++) { + u64 nls; + u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); + u64 pde = be64_to_cpu(*pdep); + + TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), + "PDE not present at level: %u gva: 0x%lx pde:0x%lx\n", + level, gva, pde); + pt = pde & PDE_PT_MASK; + } + + ptep = virt_get_pte(vm, pt, gva, level, NULL); + pte = be64_to_cpu(*ptep); + + TEST_ASSERT(pte, + "PTE not present at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + TEST_ASSERT((pte & PTE_VALID) && (pte & PTE_LEAF), + "PTE not valid at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + return (pte & PTE_PAGE_MASK) + (gva & (vm->page_size - 1)); +} + +static void virt_dump_pt(FILE *stream, struct kvm_vm *vm, gpa_t pt, + gva_t va, int level, u8 indent) +{ + int size, idx; + + size = 1U << (pt_shift(vm, level) + 3); + + for (idx = 0; idx < size; idx += 8, va += pt_entry_coverage(vm, level)) { + u64 *page_table = addr_gpa2hva(vm, pt + idx); + u64 pte = be64_to_cpu(*page_table); + + if (!(pte & PTE_VALID)) + continue; + + if (pte & PTE_LEAF) { + fprintf(stream, + "%*s PTE[%d] gVA:0x%016lx -> gRA:0x%016llx\n", + indent, "", idx / 8, va, pte & PTE_PAGE_MASK); + } else { + fprintf(stream, "%*sPDE%d[%d] gVA:0x%016lx\n", + indent, "", level, idx / 8, va); + virt_dump_pt(stream, vm, pte & PDE_PT_MASK, va, + level + 1, indent + 2); + } + } + +} + +void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent) +{ + gpa_t pt = vm->mmu.pgd; + + if (!vm->mmu.pgd_created) + return; + + virt_dump_pt(stream, vm, pt, 0, 1, indent); +} + +static unsigned long get_r2(void) +{ + unsigned long r2; + + asm("mr %0,%%r2" : "=r"(r2)); + + return r2; +} + +/* + * Under ELFv1 a function pointer is the address of a descriptor holding the + * entry point and the TOC, not the entry point itself. ELFv2 dropped + * descriptors, i.e. the pointer is already the entry point. Dereference the + * descriptor as needed so that the guest starts executing code and not data. + */ +static uintptr_t function_entry(void *fn) +{ +#if defined(_CALL_ELF) && _CALL_ELF == 2 + return (uintptr_t)fn; +#else + return ((const u64 *)fn)[0]; +#endif +} + +void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) +{ + uintptr_t entry = function_entry(guest_code); + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + regs.pc = entry; + regs.gpr[12] = entry; + vcpu_regs_set(vcpu, ®s); +} + +struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) +{ + const size_t stack_size = SZ_64K; + gva_t stack_vaddr, ex_regs_vaddr; + gpa_t ex_regs_paddr; + struct ex_regs *ex_regs; + struct kvm_regs regs; + struct kvm_vcpu *vcpu; + u64 lpcr; + + stack_vaddr = __vm_alloc(vm, stack_size, + DEFAULT_GUEST_STACK_VADDR_MIN, + MEM_REGION_DATA); + + ex_regs_vaddr = __vm_alloc(vm, stack_size, + DEFAULT_GUEST_STACK_VADDR_MIN, + MEM_REGION_DATA); + ex_regs_paddr = addr_gva2gpa(vm, ex_regs_vaddr); + ex_regs = addr_gpa2hva(vm, ex_regs_paddr); + ex_regs->gva = ex_regs_vaddr; + + vcpu = __vm_vcpu_add(vm, vcpu_id); + + vcpu_enable_cap(vcpu, KVM_CAP_PPC_PAPR, 1); + + /* Setup guest registers */ + vcpu_regs_get(vcpu, ®s); + lpcr = vcpu_get_reg(vcpu, KVM_REG_PPC_LPCR_64); + + regs.gpr[1] = stack_vaddr + stack_size - 256; + regs.gpr[2] = (uintptr_t)get_r2(); + regs.gpr[13] = (uintptr_t)ex_regs_paddr; + + regs.msr = MSR_SF | MSR_VEC | MSR_VSX | MSR_FP | + MSR_ME | MSR_IR | MSR_DR | MSR_RI; + + if (BYTE_ORDER == LITTLE_ENDIAN) { + regs.msr |= MSR_LE; + lpcr |= LPCR_ILE; + } else { + lpcr &= ~LPCR_ILE; + } + + vcpu_regs_set(vcpu, ®s); + vcpu_set_reg(vcpu, KVM_REG_PPC_LPCR_64, lpcr); + + return vcpu; +} + +void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) +{ + va_list ap; + struct kvm_regs regs; + int i; + + TEST_ASSERT(num >= 1 && num <= 5, "Unsupported number of args: %u\n", + num); + + va_start(ap, num); + vcpu_regs_get(vcpu, ®s); + + for (i = 0; i < num; i++) + regs.gpr[i + 3] = va_arg(ap, u64); + + vcpu_regs_set(vcpu, ®s); + va_end(ap); +} + +void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) +{ + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + + fprintf(stream, "%*sNIA: 0x%016llx MSR: 0x%016llx\n", + indent, "", regs.pc, regs.msr); + fprintf(stream, "%*sLR: 0x%016llx CTR :0x%016llx\n", + indent, "", regs.lr, regs.ctr); + fprintf(stream, "%*sCR: 0x%08llx XER :0x%016llx\n", + indent, "", regs.cr, regs.xer); +} + +void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) +{ + size_t excp_size = __interrupts_end - __interrupts_start; + size_t excp_pages = (excp_size + vm->page_size - 1) / vm->page_size; + gpa_t excp_paddr; + void *mem; + + /* + * Guest interrupts are taken in real-mode, i.e. the vectors must live + * at gPA 0. Use the innermost allocator to override the minimum GPA + * that MEM_REGION_DATA would otherwise impose. + */ + excp_paddr = ____vm_phy_pages_alloc(vm, excp_pages, 0, + vm->memslots[MEM_REGION_DATA], + vm_arch_has_protected_memory(vm), + false); + + TEST_ASSERT(excp_paddr == 0, + "Interrupt vectors not allocated at gPA address 0: (0x%lx)", + excp_paddr); + + mem = addr_gpa2hva(vm, excp_paddr); + memcpy(mem, __interrupts_start, excp_size); +} + +void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) +{ + struct ucall uc; + + if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { + gpa_t ex_regs_paddr; + struct ex_regs *ex_regs; + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + ex_regs_paddr = (gpa_t)regs.gpr[13]; + ex_regs = addr_gpa2hva(vcpu->vm, ex_regs_paddr); + + TEST_FAIL("Unexpected interrupt in guest NIA:0x%016lx MSR:0x%016lx TRAP:0x%04x", + ex_regs->nia, ex_regs->msr, ex_regs->trap); + } +} + +struct handler { + void (*fn)(struct ex_regs *regs); + int trap; +}; + +#define NR_HANDLERS 10 +static struct handler handlers[NR_HANDLERS]; + +void route_interrupt(struct ex_regs *regs) +{ + int i; + + for (i = 0; i < NR_HANDLERS; i++) { + if (handlers[i].trap == regs->trap) { + handlers[i].fn(regs); + return; + } + } + + ucall(UCALL_UNHANDLED, 0); +} + +void vm_install_exception_handler(struct kvm_vm *vm, int trap, + void (*fn)(struct ex_regs *)) +{ + int i; + + for (i = 0; i < NR_HANDLERS; i++) { + if (!handlers[i].trap || handlers[i].trap == trap) { + if (fn == NULL) + trap = 0; /* Clear handler */ + handlers[i].trap = trap; + handlers[i].fn = fn; + sync_global_to_guest(vm, handlers[i]); + return; + } + } + + TEST_FAIL("Out of exception handlers"); +} + +void kvm_selftest_arch_init(void) +{ + TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX)); + + /* + * powerpc default mode is set by host page size and not static, + * so start by computing that early. + */ + guest_modes_append_default(); +}
diff --git a/tools/testing/selftests/kvm/lib/powerpc/ucall.c b/tools/testing/selftests/kvm/lib/powerpc/ucall.c new file mode 100644 index 0000000..3481a7a --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/ucall.c
@@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ucall support. A ucall is a "hypercall to host userspace". + */ +#include "kvm_util.h" +#include "ucall_common.h" +#include "hcall.h" + +void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + if (run->exit_reason == UCALL_EXIT_REASON && + run->papr_hcall.nr == H_UCALL) { + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + if (regs.gpr[4] == UCALL_R4_UCALL) + return (void *)regs.gpr[5]; + } + return NULL; +}
diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index ded5429..c175dcd 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c
@@ -69,9 +69,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages); vm->mmu.pgd_created = true; }
diff --git a/tools/testing/selftests/kvm/lib/s390/processor.c b/tools/testing/selftests/kvm/lib/s390/processor.c index a9adb37..33f49d7 100644 --- a/tools/testing/selftests/kvm/lib/s390/processor.c +++ b/tools/testing/selftests/kvm/lib/s390/processor.c
@@ -20,9 +20,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - gpa = vm_phy_pages_alloc(vm, PAGES_PER_REGION, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + gpa = vm_alloc_page_table_pages(vm, PAGES_PER_REGION); memset(addr_gpa2hva(vm, gpa), 0xff, PAGES_PER_REGION * vm->page_size); vm->mmu.pgd = gpa; @@ -38,8 +36,7 @@ static u64 virt_alloc_region(struct kvm_vm *vm, int ri) { u64 taddr; - taddr = vm_phy_pages_alloc(vm, ri < 4 ? PAGES_PER_REGION : 1, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); + taddr = vm_alloc_page_table_pages(vm, ri < 4 ? PAGES_PER_REGION : 1); memset(addr_gpa2hva(vm, taddr), 0xff, PAGES_PER_REGION * vm->page_size); return (taddr & REGION_ENTRY_ORIGIN)
diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index 4dff72f..f5f7189 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c
@@ -19,13 +19,6 @@ #include "test_util.h" #include "kvm_syscalls.h" -sigjmp_buf expect_sigbus_jmpbuf; - -void __attribute__((used)) expect_sigbus_handler(int signum) -{ - siglongjmp(expect_sigbus_jmpbuf, 1); -} - /* * Random number generator that is usable from guest code. This is the * Park-Miller LCG using standard constants. @@ -255,6 +248,21 @@ size_t get_def_hugetlb_pagesz(void) TEST_FAIL("Error in reading /proc/meminfo"); } +size_t get_free_hugepages(size_t page_size) +{ + char path[128]; + size_t free; + + snprintf(path, sizeof(path), + "/sys/kernel/mm/hugepages/hugepages-%zukB/free_hugepages", + page_size >> 10); + if (!test_sysfs_path(path)) + return 0; + + free = get_sysfs_val(path); + return free * page_size; +} + #define ANON_FLAGS (MAP_PRIVATE | MAP_ANONYMOUS) #define ANON_HUGE_FLAGS (ANON_FLAGS | MAP_HUGETLB)
diff --git a/tools/testing/selftests/kvm/lib/userfaultfd_util.c b/tools/testing/selftests/kvm/lib/userfaultfd_util.c index f7ce5a6..49eff05 100644 --- a/tools/testing/selftests/kvm/lib/userfaultfd_util.c +++ b/tools/testing/selftests/kvm/lib/userfaultfd_util.c
@@ -119,13 +119,13 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay, uffd_desc = malloc(sizeof(struct uffd_desc)); TEST_ASSERT(uffd_desc, "Failed to malloc uffd descriptor"); - uffd_desc->pipefds = calloc(sizeof(int), num_readers); + uffd_desc->pipefds = calloc(num_readers, sizeof(int)); TEST_ASSERT(uffd_desc->pipefds, "Failed to alloc pipes"); - uffd_desc->readers = calloc(sizeof(pthread_t), num_readers); + uffd_desc->readers = calloc(num_readers, sizeof(pthread_t)); TEST_ASSERT(uffd_desc->readers, "Failed to alloc reader threads"); - uffd_desc->reader_args = calloc(sizeof(struct uffd_reader_args), num_readers); + uffd_desc->reader_args = calloc(num_readers, sizeof(struct uffd_reader_args)); TEST_ASSERT(uffd_desc->reader_args, "Failed to alloc reader_args"); uffd_desc->num_readers = num_readers;
diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index e19e8b5..9bfea36 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c
@@ -33,15 +33,15 @@ __asm__( static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) { GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); GUEST_ASSERT(ept_1g_pages_supported()); *(u64 *)vmx->stack = vcpu_id; prepare_vmcs(vmx, memstress_l2_guest_entry); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); }
diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index d31fa81..8374afd 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c
@@ -26,6 +26,8 @@ bool host_cpu_is_amd; bool host_cpu_is_intel; bool host_cpu_is_hygon; bool host_cpu_is_amd_compatible; +bool host_cpu_is_zhaoxin; +bool host_cpu_is_intel_compatible; bool is_forced_emulation_enabled; u64 guest_tsc_khz; struct kvm_mmu guest_mmu; @@ -158,7 +160,7 @@ static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent) bool kvm_is_tdp_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("ept"); else return get_kvm_amd_param_bool("npt"); @@ -529,7 +531,10 @@ void tdp_identity_map_default_memslots(struct kvm_vm *vm) /* Only memslot 0 is mapped here, ensure it's the only one being used */ for (s = 0; s < NR_MEM_REGIONS; s++) - TEST_ASSERT_EQ(vm->memslots[s], 0); + TEST_ASSERT(!vm->memslots[s] || + vm->memslots[s] == KVM_INVALID_MEMSLOT, + "Unhandled memslot '%u' for type '%u'", + vm->memslots[s], s); i = (region->region.guest_phys_addr >> vm->page_shift) - 1; last = i + (region->region.memory_size >> vm->page_shift); @@ -819,6 +824,8 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) sync_global_to_guest(vm, host_cpu_is_amd); sync_global_to_guest(vm, host_cpu_is_hygon); sync_global_to_guest(vm, host_cpu_is_amd_compatible); + sync_global_to_guest(vm, host_cpu_is_zhaoxin); + sync_global_to_guest(vm, host_cpu_is_intel_compatible); sync_global_to_guest(vm, is_forced_emulation_enabled); sync_global_to_guest(vm, pmu_errata_mask); @@ -1444,6 +1451,8 @@ void kvm_selftest_arch_init(void) host_cpu_is_amd = this_cpu_is_amd(); host_cpu_is_hygon = this_cpu_is_hygon(); host_cpu_is_amd_compatible = host_cpu_is_amd || host_cpu_is_hygon; + host_cpu_is_zhaoxin = this_cpu_is_zhaoxin(); + host_cpu_is_intel_compatible = host_cpu_is_intel || host_cpu_is_zhaoxin; is_forced_emulation_enabled = kvm_is_forced_emulation_enabled(); kvm_init_pmu_errata(); @@ -1465,13 +1474,13 @@ bool kvm_arch_has_default_irqchip(void) return true; } -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size) { - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, smram_gpa, - SMRAM_MEMSLOT, SMRAM_PAGES, 0); - TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa, - SMRAM_MEMSLOT) == smram_gpa, + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + smram_gpa, SMRAM_MEMSLOT, SMRAM_PAGES); + + TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, MEM_REGION_TEST_EXTRA) == smram_gpa, "Could not allocate guest physical addresses for SMRAM"); memset(addr_gpa2hva(vm, smram_gpa), 0x0, SMRAM_SIZE);
diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 089e1a8..5a06746 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c
@@ -125,7 +125,7 @@ vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva) return vmx; } -bool prepare_for_vmx_operation(struct vmx_pages *vmx) +void prepare_for_vmx_operation(struct vmx_pages *vmx) { u64 feature_control; u64 required; @@ -163,28 +163,75 @@ bool prepare_for_vmx_operation(struct vmx_pages *vmx) /* Enter VMX root operation. */ *(u32 *)(vmx->vmxon) = vmcs_revision(); - if (vmxon(vmx->vmxon_gpa)) - return false; - - return true; + vmxon(vmx->vmxon_gpa); } -bool load_vmcs(struct vmx_pages *vmx) +void load_vmcs(struct vmx_pages *vmx) { /* Load a VMCS. */ *(u32 *)(vmx->vmcs) = vmcs_revision(); - if (vmclear(vmx->vmcs_gpa)) - return false; + vmclear(vmx->vmcs_gpa); - if (vmptrld(vmx->vmcs_gpa)) - return false; + vmptrld(vmx->vmcs_gpa); /* Setup shadow VMCS, do not load it yet. */ *(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul; - if (vmclear(vmx->shadow_vmcs_gpa)) - return false; + vmclear(vmx->shadow_vmcs_gpa); +} - return true; +#define __BUILD_VMX_VM_ENTRY_HELPER(insn, prefix, vmwrite_insn, vmwrite_operand, \ + __host_rsp, __host_rip) \ +static int __##prefix##_##insn(void) \ +{ \ + int ret; \ + \ + __asm__ __volatile__("push $0;" \ + __stringify(vmwrite_insn) " %%rsp, %[host_rsp];" \ + "lea 1f(%%rip), %%rax;" \ + __stringify(vmwrite_insn) " %%rax, %[host_rip];" \ + VMX_SWITCH_GPRS_ASM \ + __stringify(insn)";" \ + "incq (%%rsp);" \ + "1: ;" \ + VMX_SWITCH_GPRS_ASM \ + "pop %%rax;" \ + : [ret]"=&a"(ret) \ + : [host_rsp]__stringify(vmwrite_operand)(__host_rsp), \ + [host_rip]__stringify(vmwrite_operand)(__host_rip), \ + GUEST_REGS_OFFSETS \ + : "memory", "cc"); \ + return ret; \ +} + +#define BUILD_VMX_VM_ENTRY_HELPER(insn) \ + __BUILD_VMX_VM_ENTRY_HELPER(insn, _, vmwrite, r, (u64)HOST_RSP, (u64)HOST_RIP) \ + __BUILD_VMX_VM_ENTRY_HELPER(insn, __evmcs, mov, m, \ + current_evmcs->host_rsp, current_evmcs->host_rip) + +BUILD_VMX_VM_ENTRY_HELPER(vmlaunch) +BUILD_VMX_VM_ENTRY_HELPER(vmresume) + +int __vmlaunch(void) +{ + if (enable_evmcs) { + current_evmcs->hv_clean_fields = 0; + return ____evmcs_vmlaunch(); + } + + return ____vmlaunch(); +} + +int __vmresume(void) +{ + if (enable_evmcs) { + /* HOST_RIP */ + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; + /* HOST_RSP */ + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; + return ____evmcs_vmresume(); + } + + return ____vmresume(); } static bool ept_vpid_cap_supported(u64 mask) @@ -204,8 +251,8 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) { u32 sec_exec_ctl = 0; - vmwrite(VIRTUAL_PROCESSOR_ID, 0); - vmwrite(POSTED_INTR_NV, 0); + __vmwrite(VIRTUAL_PROCESSOR_ID, 0); + __vmwrite(POSTED_INTR_NV, 0); vmwrite(PIN_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PINBASED_CTLS)); @@ -222,7 +269,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) sec_exec_ctl |= SECONDARY_EXEC_ENABLE_EPT; } - if (!vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl)) + if (!__vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl)) vmwrite(CPU_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); else { @@ -242,16 +289,16 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) VM_ENTRY_IA32E_MODE); /* 64-bit guest */ vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0); vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0); - vmwrite(TPR_THRESHOLD, 0); + __vmwrite(TPR_THRESHOLD, 0); vmwrite(CR0_GUEST_HOST_MASK, 0); vmwrite(CR4_GUEST_HOST_MASK, 0); vmwrite(CR0_READ_SHADOW, get_cr0()); vmwrite(CR4_READ_SHADOW, get_cr4()); - vmwrite(MSR_BITMAP, vmx->msr_gpa); - vmwrite(VMREAD_BITMAP, vmx->vmread_gpa); - vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa); + __vmwrite(MSR_BITMAP, vmx->msr_gpa); + __vmwrite(VMREAD_BITMAP, vmx->vmread_gpa); + __vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa); } /* @@ -261,7 +308,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) */ static inline void init_vmcs_host_state(void) { - u32 exit_controls = vmreadz(VM_EXIT_CONTROLS); + u32 exit_controls = vmread(VM_EXIT_CONTROLS); vmwrite(HOST_ES_SELECTOR, get_es()); vmwrite(HOST_CS_SELECTOR, get_cs()); @@ -301,23 +348,23 @@ static inline void init_vmcs_host_state(void) */ static inline void init_vmcs_guest_state(void *rip, void *rsp) { - vmwrite(GUEST_ES_SELECTOR, vmreadz(HOST_ES_SELECTOR)); - vmwrite(GUEST_CS_SELECTOR, vmreadz(HOST_CS_SELECTOR)); - vmwrite(GUEST_SS_SELECTOR, vmreadz(HOST_SS_SELECTOR)); - vmwrite(GUEST_DS_SELECTOR, vmreadz(HOST_DS_SELECTOR)); - vmwrite(GUEST_FS_SELECTOR, vmreadz(HOST_FS_SELECTOR)); - vmwrite(GUEST_GS_SELECTOR, vmreadz(HOST_GS_SELECTOR)); + vmwrite(GUEST_ES_SELECTOR, vmread(HOST_ES_SELECTOR)); + vmwrite(GUEST_CS_SELECTOR, vmread(HOST_CS_SELECTOR)); + vmwrite(GUEST_SS_SELECTOR, vmread(HOST_SS_SELECTOR)); + vmwrite(GUEST_DS_SELECTOR, vmread(HOST_DS_SELECTOR)); + vmwrite(GUEST_FS_SELECTOR, vmread(HOST_FS_SELECTOR)); + vmwrite(GUEST_GS_SELECTOR, vmread(HOST_GS_SELECTOR)); vmwrite(GUEST_LDTR_SELECTOR, 0); - vmwrite(GUEST_TR_SELECTOR, vmreadz(HOST_TR_SELECTOR)); - vmwrite(GUEST_INTR_STATUS, 0); - vmwrite(GUEST_PML_INDEX, 0); + vmwrite(GUEST_TR_SELECTOR, vmread(HOST_TR_SELECTOR)); + __vmwrite(GUEST_INTR_STATUS, 0); + __vmwrite(GUEST_PML_INDEX, 0); vmwrite(VMCS_LINK_POINTER, -1ll); vmwrite(GUEST_IA32_DEBUGCTL, 0); - vmwrite(GUEST_IA32_PAT, vmreadz(HOST_IA32_PAT)); - vmwrite(GUEST_IA32_EFER, vmreadz(HOST_IA32_EFER)); - vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, - vmreadz(HOST_IA32_PERF_GLOBAL_CTRL)); + __vmwrite(GUEST_IA32_PAT, vmread(HOST_IA32_PAT)); + __vmwrite(GUEST_IA32_EFER, vmread(HOST_IA32_EFER)); + __vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, + vmread(HOST_IA32_PERF_GLOBAL_CTRL)); vmwrite(GUEST_ES_LIMIT, -1); vmwrite(GUEST_CS_LIMIT, -1); @@ -330,42 +377,42 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_GDTR_LIMIT, 0xffff); vmwrite(GUEST_IDTR_LIMIT, 0xffff); vmwrite(GUEST_ES_AR_BYTES, - vmreadz(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_CS_AR_BYTES, 0xa09b); vmwrite(GUEST_SS_AR_BYTES, 0xc093); vmwrite(GUEST_DS_AR_BYTES, - vmreadz(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_FS_AR_BYTES, - vmreadz(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_GS_AR_BYTES, - vmreadz(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_LDTR_AR_BYTES, 0x10000); vmwrite(GUEST_TR_AR_BYTES, 0x8b); vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0); vmwrite(GUEST_ACTIVITY_STATE, 0); - vmwrite(GUEST_SYSENTER_CS, vmreadz(HOST_IA32_SYSENTER_CS)); - vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); + vmwrite(GUEST_SYSENTER_CS, vmread(HOST_IA32_SYSENTER_CS)); + __vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); - vmwrite(GUEST_CR0, vmreadz(HOST_CR0)); - vmwrite(GUEST_CR3, vmreadz(HOST_CR3)); - vmwrite(GUEST_CR4, vmreadz(HOST_CR4)); + vmwrite(GUEST_CR0, vmread(HOST_CR0)); + vmwrite(GUEST_CR3, vmread(HOST_CR3)); + vmwrite(GUEST_CR4, vmread(HOST_CR4)); vmwrite(GUEST_ES_BASE, 0); vmwrite(GUEST_CS_BASE, 0); vmwrite(GUEST_SS_BASE, 0); vmwrite(GUEST_DS_BASE, 0); - vmwrite(GUEST_FS_BASE, vmreadz(HOST_FS_BASE)); - vmwrite(GUEST_GS_BASE, vmreadz(HOST_GS_BASE)); + vmwrite(GUEST_FS_BASE, vmread(HOST_FS_BASE)); + vmwrite(GUEST_GS_BASE, vmread(HOST_GS_BASE)); vmwrite(GUEST_LDTR_BASE, 0); - vmwrite(GUEST_TR_BASE, vmreadz(HOST_TR_BASE)); - vmwrite(GUEST_GDTR_BASE, vmreadz(HOST_GDTR_BASE)); - vmwrite(GUEST_IDTR_BASE, vmreadz(HOST_IDTR_BASE)); + vmwrite(GUEST_TR_BASE, vmread(HOST_TR_BASE)); + vmwrite(GUEST_GDTR_BASE, vmread(HOST_GDTR_BASE)); + vmwrite(GUEST_IDTR_BASE, vmread(HOST_IDTR_BASE)); vmwrite(GUEST_DR7, 0x400); vmwrite(GUEST_RSP, (u64)rsp); vmwrite(GUEST_RIP, (u64)rip); vmwrite(GUEST_RFLAGS, X86_EFLAGS_FIXED); vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0); - vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP)); - vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP)); + vmwrite(GUEST_SYSENTER_ESP, vmread(HOST_IA32_SYSENTER_ESP)); + vmwrite(GUEST_SYSENTER_EIP, vmread(HOST_IA32_SYSENTER_EIP)); } void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) @@ -375,19 +422,19 @@ void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) init_vmcs_guest_state(guest_rip, vmx->stack); } -bool kvm_cpu_has_ept(void) +bool kvm_cpu_has_secondary_exec_control(u32 ctrl) { - u64 ctrl; + u64 ctrl_msr; if (!kvm_cpu_has(X86_FEATURE_VMX)) return false; - ctrl = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; - if (!(ctrl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) + ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; + if (!(ctrl_msr & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) return false; - ctrl = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; - return ctrl & SECONDARY_EXEC_ENABLE_EPT; + ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; + return ctrl_msr & ctrl; } void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm)
diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index e129961..db2fd3c 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c
@@ -30,7 +30,7 @@ #define MEM_EXTRA_SIZE SZ_64K #define MEM_SIZE (SZ_512M + MEM_EXTRA_SIZE) -#define MEM_GPA SZ_256M +#define MEM_GPA SZ_512M #define MEM_AUX_GPA MEM_GPA #define MEM_SYNC_GPA MEM_AUX_GPA #define MEM_TEST_GPA (MEM_AUX_GPA + MEM_EXTRA_SIZE) @@ -152,7 +152,7 @@ static void *vcpu_worker(void *__data) case UCALL_SYNC: TEST_ASSERT(uc.args[1] == 0, "Unexpected sync ucall, got %lx", - (ulong)uc.args[1]); + (unsigned long)uc.args[1]); sem_post(&vcpu_ready); continue; case UCALL_NONE: @@ -339,7 +339,8 @@ static bool prepare_vm(struct vm_data *data, int nslots, u64 *maxslots, if (slot == data->nslots) npages += rempages; - gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot); + gpa = ____vm_phy_pages_alloc(data->vm, npages, guest_addr, slot, + false, false); TEST_ASSERT(gpa == guest_addr, "vm_phy_pages_alloc() failed");
diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c index c57631a..3082fe0 100644 --- a/tools/testing/selftests/kvm/pre_fault_memory_test.c +++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c
@@ -12,19 +12,29 @@ #include <processor.h> #include <pthread.h> #include <ucall_common.h> +#include <guest_modes.h> /* Arbitrarily chosen values */ -#define TEST_SIZE (SZ_2M + PAGE_SIZE) -#define TEST_NPAGES (TEST_SIZE / PAGE_SIZE) +#define TEST_BASE_SIZE SZ_2M #define TEST_SLOT 10 +/* Storage of test info to share with guest code */ +struct test_config { + u64 page_size; + u64 test_size; + u64 test_num_pages; +}; + +static struct test_config test_config; + static void guest_code(u64 base_gva) { volatile u64 val __used; + struct test_config *config = &test_config; int i; - for (i = 0; i < TEST_NPAGES; i++) { - u64 *src = (u64 *)(base_gva + i * PAGE_SIZE); + for (i = 0; i < config->test_num_pages; i++) { + u64 *src = (u64 *)(base_gva + i * config->page_size); val = *src; } @@ -36,6 +46,7 @@ struct slot_worker_data { struct kvm_vm *vm; gpa_t gpa; u32 flags; + enum vm_mem_backing_src_type mem_backing_src; bool worker_ready; bool prefault_ready; bool recreate_slot; @@ -56,14 +67,16 @@ static void *delete_slot_worker(void *__data) while (!READ_ONCE(data->recreate_slot)) cpu_relax(); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, data->gpa, - TEST_SLOT, TEST_NPAGES, data->flags); + vm_userspace_mem_region_add(vm, data->mem_backing_src, data->gpa, + TEST_SLOT, test_config.test_num_pages, data->flags); return NULL; } static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, - u64 size, u64 expected_left, bool private) + u64 size, u64 expected_left, + enum vm_mem_backing_src_type mem_backing_src, + bool private) { struct kvm_pre_fault_memory range = { .gpa = base_gpa + offset, @@ -74,6 +87,7 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, .vm = vcpu->vm, .gpa = base_gpa, .flags = private ? KVM_MEM_GUEST_MEMFD : 0, + .mem_backing_src = mem_backing_src, }; bool slot_recreated = false; pthread_t slot_worker; @@ -150,8 +164,8 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, /* * Assert success if prefaulting the entire range should succeed, i.e. * complete with no bytes remaining. Otherwise prefaulting should have - * failed due to ENOENT (due to RET_PF_EMULATE for emulated MMIO when - * no memslot exists). + * failed due to ENOENT (no memslot exists for the GPA; on x86 this + * surfaces via RET_PF_EMULATE). */ if (!expected_left) TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm); @@ -160,39 +174,82 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm); } -static void __test_pre_fault_memory(unsigned long vm_type, bool private) +struct test_params { + unsigned long vm_type; + bool private; + enum vm_mem_backing_src_type mem_backing_src; +}; + +static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) { - gpa_t gpa, gva, alignment, guest_page_size; + gpa_t gpa, gva, alignment, guest_page_size, host_page_size; + gpa_t backing_src_pagesz, mem_page_size; + struct test_params *p = arg; const struct vm_shape shape = { - .mode = VM_MODE_DEFAULT, - .type = vm_type, + .mode = guest_mode, + .type = p->vm_type, }; struct kvm_vcpu *vcpu; + struct kvm_run *run; struct kvm_vm *vm; struct ucall uc; + pr_info("Testing guest mode: %s\n", vm_guest_mode_string(guest_mode)); + pr_info("Testing memory backing src type: %s\n", + vm_mem_backing_src_alias(p->mem_backing_src)->name); + vm = vm_create_shape_with_one_vcpu(shape, &vcpu, guest_code); - alignment = guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size; - gpa = (vm->max_gfn - TEST_NPAGES) * guest_page_size; + guest_page_size = vm_guest_mode_params[guest_mode].page_size; + host_page_size = getpagesize(); + backing_src_pagesz = get_backing_src_pagesz(p->mem_backing_src); + mem_page_size = max(host_page_size, backing_src_pagesz); + + test_config.page_size = guest_page_size; + test_config.test_size = align_up(TEST_BASE_SIZE + test_config.page_size, + mem_page_size); + test_config.test_num_pages = vm_calc_num_guest_pages(vm->mode, test_config.test_size); + + gpa = (vm->max_gfn - test_config.test_num_pages) * test_config.page_size; alignment = SZ_2M; + alignment = max(alignment, mem_page_size); gpa = align_down(gpa, alignment); gva = gpa & ((1ULL << (vm->va_bits - 1)) - 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa, TEST_SLOT, - TEST_NPAGES, private ? KVM_MEM_GUEST_MEMFD : 0); - virt_map(vm, gva, gpa, TEST_NPAGES); + vm_userspace_mem_region_add(vm, p->mem_backing_src, + gpa, TEST_SLOT, test_config.test_num_pages, + p->private ? KVM_MEM_GUEST_MEMFD : 0); + virt_map(vm, gva, gpa, test_config.test_num_pages); - if (private) - vm_mem_set_private(vm, gpa, TEST_SIZE); + if (p->private) + vm_mem_set_private(vm, gpa, test_config.test_size); - pre_fault_memory(vcpu, gpa, 0, SZ_2M, 0, private); - pre_fault_memory(vcpu, gpa, SZ_2M, PAGE_SIZE * 2, PAGE_SIZE, private); - pre_fault_memory(vcpu, gpa, TEST_SIZE, PAGE_SIZE, PAGE_SIZE, private); + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, + p->mem_backing_src, p->private); + /* Retry the same range after the first prefault attempt. */ + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, + p->mem_backing_src, p->private); + pre_fault_memory(vcpu, gpa, + test_config.test_size - host_page_size, + host_page_size * 2, host_page_size, + p->mem_backing_src, p->private); + pre_fault_memory(vcpu, gpa, test_config.test_size, + host_page_size, host_page_size, + p->mem_backing_src, p->private); vcpu_args_set(vcpu, 1, gva); + + /* Export the shared variables to the guest. */ + sync_global_to_guest(vm, test_config); + vcpu_run(vcpu); + run = vcpu->run; + TEST_ASSERT(run->exit_reason == UCALL_EXIT_REASON, + "Wanted %s, got exit reason: %u (%s)", + exit_reason_str(UCALL_EXIT_REASON), + run->exit_reason, exit_reason_str(run->exit_reason)); + switch (get_ucall(vcpu, &uc)) { case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); @@ -207,24 +264,61 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) kvm_vm_free(vm); } -static void test_pre_fault_memory(unsigned long vm_type, bool private) +static void test_pre_fault_memory(unsigned long vm_type, enum vm_mem_backing_src_type backing_src, + bool private) { + struct test_params p = { + .vm_type = vm_type, + .private = private, + .mem_backing_src = backing_src, + }; + if (vm_type && !(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(vm_type))) { pr_info("Skipping tests for vm_type 0x%lx\n", vm_type); return; } - __test_pre_fault_memory(vm_type, private); + for_each_guest_mode(__test_pre_fault_memory, &p); +} + +static void help(char *name) +{ + puts(""); + printf("usage: %s [-h] [-m mode] [-s mem-type]\n", name); + puts(""); + guest_modes_help(); + backing_src_help("-s"); + puts(""); } int main(int argc, char *argv[]) { + enum vm_mem_backing_src_type backing = DEFAULT_VM_MEM_SRC; + int opt; + + guest_modes_append_default(); + + while ((opt = getopt(argc, argv, "hm:s:")) != -1) { + switch (opt) { + case 'm': + guest_modes_cmdline(optarg); + break; + case 's': + backing = parse_backing_src_type(optarg); + break; + case 'h': + default: + help(argv[0]); + exit(0); + } + } + TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); - test_pre_fault_memory(0, false); + test_pre_fault_memory(0, backing, false); #ifdef __x86_64__ - test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, false); - test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, true); + test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, false); + test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, true); #endif return 0; }
diff --git a/tools/testing/selftests/kvm/s390/cmma_test.c b/tools/testing/selftests/kvm/s390/cmma_test.c index 15d81b2..73387cc 100644 --- a/tools/testing/selftests/kvm/s390/cmma_test.c +++ b/tools/testing/selftests/kvm/s390/cmma_test.c
@@ -102,24 +102,17 @@ static void guest_dirty_test_data(void) static void create_main_memslot(struct kvm_vm *vm) { - int i; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, MAIN_PAGE_COUNT, 0); - /* set the array of memslots to zero like __vm_create does */ - for (i = 0; i < NR_MEM_REGIONS; i++) - vm->memslots[i] = 0; + ____vm_override_mem_region(vm, MEM_REGION_CODE, 0); + ____vm_override_mem_region(vm, MEM_REGION_PT, 0); + ____vm_override_mem_region(vm, MEM_REGION_DATA, 0); } static void create_test_memslot(struct kvm_vm *vm) { - vm_userspace_mem_region_add(vm, - VM_MEM_SRC_ANONYMOUS, - TEST_DATA_START_GFN << vm->page_shift, - TEST_DATA_MEMSLOT, - TEST_DATA_PAGE_COUNT, - 0 - ); - vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; + vm_override_mem_region(vm, MEM_REGION_TEST_DATA, VM_MEM_SRC_ANONYMOUS, + TEST_DATA_START_GFN << vm->page_shift, + TEST_DATA_MEMSLOT, TEST_DATA_PAGE_COUNT); } static void create_memslots(struct kvm_vm *vm)
diff --git a/tools/testing/selftests/kvm/s390/irq_injection.c b/tools/testing/selftests/kvm/s390/irq_injection.c new file mode 100644 index 0000000..75e0528 --- /dev/null +++ b/tools/testing/selftests/kvm/s390/irq_injection.c
@@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Test for KVM_S390_IRQ - s390x IRQ injection API + * + * Verifies that VM-level interrupt types are correctly rejected when + * injected via the vCPU file descriptor, and that invalid arguments + * to vCPU-level interrupts are also rejected. + * + * Copyright IBM Corp. 2026 + * Author: Gabriella Seifert <gseifert@linux.ibm.com> + */ +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include "kvm_util.h" +#include "test_util.h" +#include <linux/kvm.h> + +static void test_vm_irq_on_vcpu_fails(struct kvm_vcpu *vcpu, __u64 type) +{ + struct kvm_s390_irq irq = { .type = type }; + int rc; + + rc = __vcpu_ioctl(vcpu, KVM_S390_IRQ, &irq); + ksft_test_result(rc == -1 && errno == EINVAL, + "VM on VCPU IRQ type 0x%llx: expected %d, got %d\n", + type, EINVAL, errno); +} + +static void test_vcpu_irq_bad_param_fails(struct kvm_vcpu *vcpu, + struct kvm_s390_irq *irq) +{ + int rc; + + rc = __vcpu_ioctl(vcpu, KVM_S390_IRQ, irq); + ksft_test_result(rc == -1 && errno == EINVAL, + "VCPU IRQ bad param type 0x%llx: expected %d got %d\n", + irq->type, EINVAL, errno); +} + +int main(void) +{ + struct kvm_s390_irq irq; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + ksft_print_header(); + ksft_set_plan(6); + vm = vm_create_with_one_vcpu(&vcpu, NULL); + + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_VIRTIO); + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_SERVICE); + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_IO_MIN); + + memset(&irq, 0, sizeof(irq)); + irq.type = KVM_S390_INT_EMERGENCY; + irq.u.emerg.code = 0xffff; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + memset(&irq, 0, sizeof(irq)); + irq.type = KVM_S390_INT_EXTERNAL_CALL; + irq.u.extcall.code = 0xffff; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + memset(&irq, 0, sizeof(irq)); + irq.type = 0xdeadbeef; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + kvm_vm_free(vm); + ksft_finished(); +}
diff --git a/tools/testing/selftests/kvm/s390/irq_routing.c b/tools/testing/selftests/kvm/s390/irq_routing.c index f383928..cd26cc1 100644 --- a/tools/testing/selftests/kvm/s390/irq_routing.c +++ b/tools/testing/selftests/kvm/s390/irq_routing.c
@@ -36,7 +36,7 @@ static void test(void) }; vm = vm_create_with_one_vcpu(&vcpu, guest_code); - mem = vm_phy_pages_alloc(vm, 2, 4096 * 42, 0); + mem = ____vm_phy_pages_alloc(vm, 2, 4096 * 42, 0, false, false); routing = kvm_gsi_routing_create(); routing->nr = 1;
diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index dffe965..bfa5ac8 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c
@@ -116,15 +116,15 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread, vm = vm_create_with_one_vcpu(vcpu, guest_code); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, - MEM_REGION_GPA, MEM_REGION_SLOT, - MEM_REGION_SIZE / getpagesize(), 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS_THP, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / getpagesize()); /* * Allocate and map two pages so that the GPA accessed by guest_code() * stays valid across the memslot move. */ - gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT); + gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2);
diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 785d19f9..82b94a82 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c
@@ -76,12 +76,12 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); - vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED); + vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (unsigned long)st_gva[i] | KVM_MSR_ENABLED); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" steal: %lld\n", st->steal); @@ -105,7 +105,7 @@ static void check_steal_time_uapi(void) vm = vm_create_with_one_vcpu(&vcpu, NULL); ret = _vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, - (ulong)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK); + (unsigned long)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK); TEST_ASSERT(ret == 0, "Bad GPA didn't fail"); kvm_vm_free(vm); @@ -154,7 +154,7 @@ static void guest_code(int cpu) status = smccc(PV_TIME_ST, 0); GUEST_ASSERT_NE(status, -1); - GUEST_ASSERT_EQ(status, (ulong)st_gva[cpu]); + GUEST_ASSERT_EQ(status, (unsigned long)st_gva[cpu]); st = (struct st_time *)status; GUEST_SYNC(0); @@ -192,13 +192,13 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) /* ST_GPA_BASE is identity mapped */ WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); - st_ipa = (ulong)st_gva[i]; + st_ipa = (unsigned long)st_gva[i]; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct st_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct st_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" rev: %d\n", st->rev); @@ -222,11 +222,11 @@ static void check_steal_time_uapi(void) vm = vm_create_steal_time(1, NULL, &vcpu); vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev); - st_ipa = (ulong)ST_GPA_BASE | 1; + st_ipa = (unsigned long)ST_GPA_BASE | 1; ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); TEST_ASSERT(ret == -1 && errno == EINVAL, "Bad IPA didn't report EINVAL"); - st_ipa = (ulong)ST_GPA_BASE; + st_ipa = (unsigned long)ST_GPA_BASE; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -315,7 +315,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct sta_struct *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct sta_struct *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); int i; pr_info("VCPU%d:\n", vcpu_idx); @@ -451,7 +451,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" steal: %lld\n", st->steal);
diff --git a/tools/testing/selftests/kvm/x86/amx_test.c b/tools/testing/selftests/kvm/x86/amx_test.c index 4e63da2..66b6e44 100644 --- a/tools/testing/selftests/kvm/x86/amx_test.c +++ b/tools/testing/selftests/kvm/x86/amx_test.c
@@ -325,7 +325,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } break; case UCALL_DONE:
diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index 845cb68..1ca6495 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
@@ -70,8 +70,8 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); @@ -80,10 +80,10 @@ static void l1_vmx_code(struct vmx_pages *vmx) * in the VMCS by prepare_vmcs()), as MSR exiting mandatory on Intel. */ vmwrite(CPU_BASED_VM_EXEC_CONTROL, - vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); + vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmlaunch()); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmlaunch(); } static void guest_code(void *nested_test_data)
diff --git a/tools/testing/selftests/kvm/x86/cpuid_test.c b/tools/testing/selftests/kvm/x86/cpuid_test.c index ef0ddd2..dcd99a0 100644 --- a/tools/testing/selftests/kvm/x86/cpuid_test.c +++ b/tools/testing/selftests/kvm/x86/cpuid_test.c
@@ -128,7 +128,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage + 1, "Stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); return; case UCALL_DONE: return;
diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c index aa7f3b40..4a65eb0 100644 --- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c +++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
@@ -55,11 +55,11 @@ static void guest_code(struct vmx_pages *vmx_pages, enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); evmcs_enable(); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_evmcs(hv_pages)); + prepare_for_vmx_operation(vmx_pages); + load_evmcs(hv_pages); prepare_vmcs(vmx_pages, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 exits via vmcall if test fails */ sync_with_host(2);
diff --git a/tools/testing/selftests/kvm/x86/feature_msrs_test.c b/tools/testing/selftests/kvm/x86/feature_msrs_test.c index 1585507..d56d0ddb 100644 --- a/tools/testing/selftests/kvm/x86/feature_msrs_test.c +++ b/tools/testing/selftests/kvm/x86/feature_msrs_test.c
@@ -59,7 +59,7 @@ static void test_feature_msr(u32 msr) * but initializes the vCPU's revision ID to an arbitrary value. */ if (msr == MSR_IA32_UCODE_REV) - reset_value = host_cpu_is_intel ? 0x100000000ULL : 0x01000065; + reset_value = host_cpu_is_intel_compatible ? 0x100000000ULL : 0x01000065; /* * For quirked MSRs, KVM's ABI is to initialize the vCPU's value to the
diff --git a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c index 4931ec2..f11fcc5 100644 --- a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c +++ b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c
@@ -48,7 +48,7 @@ static void guest_main(void) const u8 *other_hypercall_insn; u64 ret; - if (host_cpu_is_intel) { + if (host_cpu_is_intel_compatible) { native_hypercall_insn = vmx_vmcall; other_hypercall_insn = svm_vmmcall; } else if (host_cpu_is_amd_compatible) {
diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c new file mode 100644 index 0000000..e1c84df --- /dev/null +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c
@@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Google LLC. + */ +#include <pthread.h> +#include <sys/mman.h> +#include <time.h> +#include <unistd.h> + +#include <linux/align.h> +#include <linux/kvm.h> +#include <linux/sizes.h> + +#include "kvm_util.h" +#include "kvm_syscalls.h" +#include "kselftest_harness.h" +#include "test_util.h" +#include "ucall_common.h" + +FIXTURE(gmem_conversions) { + struct kvm_vcpu *vcpu; + int gmem_fd; + /* HVA of the first byte of the memory mmap()-ed from gmem_fd. */ + char *mem; +}; + +typedef FIXTURE_DATA(gmem_conversions) test_data_t; + +FIXTURE_SETUP(gmem_conversions) { } + +static size_t page_size; + +static void guest_do_rmw(void); +#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL + +/* + * Defer setup until the individual test is invoked so that tests can specify + * the number of pages and flags for the guest_memfd instance. + */ +static void gmem_conversions_do_setup(test_data_t *t, int nr_pages, + int gmem_flags) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + /* + * Use high GPA above APIC_DEFAULT_PHYS_BASE to avoid clashing with + * APIC_DEFAULT_PHYS_BASE. + */ + const gpa_t gpa = SZ_4G; + const u32 slot = 1; + struct kvm_vm *vm; + + vm = __vm_create_shape_with_one_vcpu(shape, &t->vcpu, nr_pages, guest_do_rmw); + + vm_mem_add(vm, VM_MEM_SRC_SHMEM, gpa, slot, nr_pages, + KVM_MEM_GUEST_MEMFD, -1, 0, gmem_flags); + + t->gmem_fd = kvm_slot_to_fd(vm, slot); + t->mem = addr_gpa2hva(vm, gpa); + virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, gpa, nr_pages); +} + +static void gmem_conversions_do_teardown(test_data_t *t) +{ + /* Use NULL to avoid second free in FIXTURE_TEARDOWN (multipage tests). */ + if (!t->vcpu) + return; + + /* No need to close gmem_fd, it's owned by the VM structure. */ + kvm_vm_free(t->vcpu->vm); + t->vcpu = NULL; +} + +FIXTURE_TEARDOWN(gmem_conversions) +{ + gmem_conversions_do_teardown(self); +} + +/* + * In these test definition macros, __nr_pages and nr_pages is used to set up + * the total number of pages in the guest_memfd under test. This will be + * available in the test definitions as nr_pages. + */ + +#define __GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const int nr = (__nr_pages); \ + \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_##test(self, nr); \ +} \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \ + +#define GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ + __GMEM_CONVERSION_TEST(test, __nr_pages, (flags) | GUEST_MEMFD_FLAG_MMAP) + +#define __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, 0) + +#define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \ + __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1) + +#define __GMEM_CONVERSION_TEST_INIT_SHARED(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, GUEST_MEMFD_FLAG_INIT_SHARED) + +#define GMEM_CONVERSION_TEST_INIT_SHARED(test) \ + __GMEM_CONVERSION_TEST_INIT_SHARED(test, 1) + +/* + * Repeats test over nr_pages in a guest_memfd of size nr_pages, providing each + * test iteration with test_page, the index of the page under test in + * guest_memfd. test_page takes values 0..(nr_pages - 1) inclusive. + */ +#define GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(test, __nr_pages) \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const u64 flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; \ + const int nr = (__nr_pages); \ + int i; \ + \ + for (i = 0; i < nr; ++i) { \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_multipage_##test(self, nr, i); \ + gmem_conversions_do_teardown(self); \ + } \ +} \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page) + +struct guest_check_data { + void *mem; + char expected_val; + char write_val; +}; +static struct guest_check_data guest_data; + +static void guest_do_rmw(void) +{ + for (;;) { + char *mem = READ_ONCE(guest_data.mem); + + GUEST_ASSERT_EQ(READ_ONCE(*mem), READ_ONCE(guest_data.expected_val)); + WRITE_ONCE(*mem, READ_ONCE(guest_data.write_val)); + + GUEST_SYNC(0); + } +} + +static void run_guest_do_rmw(struct kvm_vcpu *vcpu, u64 pgoff, + char expected_val, char write_val) +{ + struct ucall uc; + int r; + + guest_data.mem = (void *)GUEST_MEMFD_SHARING_TEST_GVA + pgoff * page_size; + guest_data.expected_val = expected_val; + guest_data.write_val = write_val; + sync_global_to_guest(vcpu->vm, guest_data); + + do { + r = __vcpu_run(vcpu); + } while (r == -1 && errno == EINTR); + + TEST_ASSERT_EQ(r, 0); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_SYNC: + break; + default: + TEST_FAIL("Unexpected ucall %lu", uc.cmd); + } +} + +static void host_do_rmw(char *mem, u64 pgoff, char expected_val, + char write_val) +{ + TEST_ASSERT_EQ(READ_ONCE(mem[pgoff * page_size]), expected_val); + WRITE_ONCE(mem[pgoff * page_size], write_val); +} + +static void test_private(test_data_t *t, u64 pgoff, char starting_val, + char write_val) +{ + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[pgoff * page_size], write_val)); + run_guest_do_rmw(t->vcpu, pgoff, starting_val, write_val); + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[pgoff * page_size])); +} + +static void test_convert_to_private(test_data_t *t, u64 pgoff, + char starting_val, char write_val) +{ + gmem_set_private(t->gmem_fd, pgoff * page_size, page_size); + test_private(t, pgoff, starting_val, write_val); +} + +static void test_shared(test_data_t *t, u64 pgoff, char starting_val, + char host_write_val, char write_val) +{ + host_do_rmw(t->mem, pgoff, starting_val, host_write_val); + run_guest_do_rmw(t->vcpu, pgoff, host_write_val, write_val); + TEST_ASSERT_EQ(READ_ONCE(t->mem[pgoff * page_size]), write_val); +} + +static void test_convert_to_shared(test_data_t *t, u64 pgoff, + char starting_val, char host_write_val, + char write_val) +{ + gmem_set_shared(t->gmem_fd, pgoff * page_size, page_size); + test_shared(t, pgoff, starting_val, host_write_val, write_val); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private) +{ + test_private(t, 0, 0, 'A'); + test_convert_to_shared(t, 0, 'A', 'B', 'C'); + test_convert_to_private(t, 0, 'C', 'E'); +} + +GMEM_CONVERSION_TEST_INIT_SHARED(init_shared) +{ + test_shared(t, 0, 0, 'A', 'B'); + test_convert_to_private(t, 0, 'B', 'C'); + test_convert_to_shared(t, 0, 'C', 'D', 'E'); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4) +{ + int i; + + /* Get a char that varies with both i and n. */ +#define combine(x, n) (((x) << 4) + (n)) +#define i_(n) (combine(i, n)) +#define t_(n) (combine(test_page, n)) + + /* + * Start with the highest index, to catch any errors when, perhaps, the + * first page is returned even for the last index. + */ + for (i = nr_pages - 1; i >= 0; --i) + test_shared(t, i, 0, i_(0), i_(2)); + + test_convert_to_private(t, test_page, t_(2), t_(3)); + + for (i = 0; i < nr_pages; ++i) { + if (i == test_page) + test_private(t, test_page, t_(3), t_(4)); + else + test_shared(t, i, i_(2), i_(3), i_(4)); + } + + test_convert_to_shared(t, test_page, t_(4), t_(5), t_(6)); + + for (i = 0; i < nr_pages; ++i) { + char expected = i == test_page ? t_(6) : i_(4); + + test_shared(t, i, expected, i_(7), i_(8)); + } + +#undef t_ +#undef i_ +#undef combine +} + +/* + * Test that even if there are no folios yet, conversion requests are recorded + * in guest_memfd. + */ +GMEM_CONVERSION_TEST_INIT_SHARED(before_allocation_shared) +{ + test_convert_to_private(t, 0, 0, 'A'); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private) +{ + test_convert_to_shared(t, 0, 0, 'A', 'B'); +} + +/* + * Test that when some of the folios in the conversion range are allocated, + * conversion requests are handled correctly in guest_memfd. Vary the ranges + * allocated before conversion, using test_page, to cover various layouts of + * contiguous allocated and unallocated regions. + */ +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8) +{ + const int second_page_to_fault = 4; + int i; + + /* + * Fault 2 of the pages to test filemap range operations except when + * test_page == second_page_to_fault. + */ + host_do_rmw(t->mem, test_page, 0, 'A'); + if (test_page != second_page_to_fault) + host_do_rmw(t->mem, second_page_to_fault, 0, 'A'); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + for (i = 0; i < nr_pages; ++i) { + char expected = (i == test_page || i == second_page_to_fault) ? 'A' : 0; + + test_private(t, i, expected, 'B'); + } + + for (i = 0; i < nr_pages; ++i) + test_convert_to_shared(t, i, 'B', 'C', 'D'); +} + +/* Truncation should not affect shared/private status. */ +GMEM_CONVERSION_TEST_INIT_SHARED(truncate) +{ + host_do_rmw(t->mem, 0, 0, 'A'); + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + host_do_rmw(t->mem, 0, 0, 'A'); + + test_convert_to_private(t, 0, 'A', 'B'); + + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + test_private(t, 0, 0, 'A'); +} + +/* Test that shared/private memory protections work and are seen from any process. */ +GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses) +{ + enum test_state { + STATE_INIT, + STATE_CHECK_SHARED, + STATE_DONE_CHECKING_SHARED, + STATE_CHECK_PRIVATE, + STATE_DONE_CHECKING_PRIVATE, + }; + + struct sync_state { + pthread_mutex_t mutex; + pthread_cond_t cond; + enum test_state step; + } *sync; + + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + pid_t child_pid, parent_pid; + int status; + + sync = kvm_mmap(sizeof(*sync), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); + + pthread_mutexattr_init(&mattr); + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); + pthread_mutex_init(&sync->mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + pthread_condattr_init(&cattr); + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); + pthread_cond_init(&sync->cond, &cattr); + pthread_condattr_destroy(&cattr); + + sync->step = STATE_INIT; + +#define TEST_STATE_AWAIT(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + while (sync->step != (__state)) { \ + struct timespec ts, stop; \ + int ret; \ + \ + clock_gettime(CLOCK_REALTIME, &ts); \ + stop = timespec_add_ns(ts, 100 * 1000000UL); \ + \ + ret = pthread_cond_timedwait(&sync->cond, &sync->mutex, &stop); \ + if (ret == ETIMEDOUT) { \ + bool alive = (child_pid == 0) ? \ + (getppid() == parent_pid) : \ + (waitpid(child_pid, NULL, WNOHANG) == 0); \ + TEST_ASSERT(alive, "Other process exited prematurely"); \ + } else { \ + TEST_ASSERT(!ret, "pthread_cond_timedwait failed"); \ + } \ + } \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + +#define TEST_STATE_SET(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + sync->step = (__state); \ + pthread_cond_broadcast(&sync->cond); \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + + parent_pid = getpid(); + child_pid = fork(); + TEST_ASSERT(child_pid != -1, "fork failed"); + + if (child_pid == 0) { + const char inconsequential = 0xdd; + + TEST_STATE_AWAIT(STATE_CHECK_SHARED); + + /* + * This maps the pages into the child process as well, and tests + * that the conversion process will unmap the guest_memfd memory + * from all processes. + */ + host_do_rmw(t->mem, 0, 0xB, 0xC); + + TEST_STATE_SET(STATE_DONE_CHECKING_SHARED); + TEST_STATE_AWAIT(STATE_CHECK_PRIVATE); + + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[0])); + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[0], inconsequential)); + + TEST_STATE_SET(STATE_DONE_CHECKING_PRIVATE); + exit(0); + } + + test_shared(t, 0, 0, 0xA, 0xB); + + TEST_STATE_SET(STATE_CHECK_SHARED); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_SHARED); + + test_convert_to_private(t, 0, 0xC, 0xD); + + TEST_STATE_SET(STATE_CHECK_PRIVATE); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_PRIVATE); + + TEST_ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid); + TEST_ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "Child exited with unexpected status"); + + pthread_mutex_destroy(&sync->mutex); + pthread_cond_destroy(&sync->cond); + kvm_munmap(sync, sizeof(*sync)); + +#undef TEST_STATE_SET +#undef TEST_STATE_AWAIT +} + +static void test_convert_to_private_fails(test_data_t *t, u64 pgoff, + size_t nr_pages, + u64 expected_error_offset) +{ + /* +1 to make it anything but expected_error_offset. */ + u64 error_offset = expected_error_offset + 1; + u64 offset = pgoff * page_size; + int ret; + + do { + ret = __gmem_set_private(t->gmem_fd, offset, + nr_pages * page_size, &error_offset); + } while (ret == -1 && errno == EINTR); + TEST_ASSERT(ret == -1 && errno == EAGAIN, + "Wanted EAGAIN on page %lu, got %d (ret = %d)", pgoff, + errno, ret); + TEST_ASSERT_EQ(error_offset, expected_error_offset); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(elevated_refcount, 4) +{ + int i; + + pin_pages(t->mem + test_page * page_size, page_size); + + for (i = 0; i < nr_pages; i++) + test_shared(t, i, 0, 'A', 'B'); + + /* + * Converting in bulk should fail as long any page in the range has + * unexpected refcounts. + */ + test_convert_to_private_fails(t, 0, nr_pages, test_page * page_size); + + for (i = 0; i < nr_pages; i++) { + /* + * Converting page-wise should also fail as long any page in the + * range has unexpected refcounts. + */ + if (i == test_page) + test_convert_to_private_fails(t, i, 1, test_page * page_size); + else + test_convert_to_private(t, i, 'B', 'C'); + } + + unpin_pages(); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + + for (i = 0; i < nr_pages; i++) { + char expected = i == test_page ? 'B' : 'C'; + + test_private(t, i, expected, 'D'); + } +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) & + KVM_MEMORY_ATTRIBUTE_PRIVATE); + + page_size = getpagesize(); + + return test_harness_run(argc, argv); +}
diff --git a/tools/testing/selftests/kvm/x86/hyperv_clock.c b/tools/testing/selftests/kvm/x86/hyperv_clock.c index d5d7796..7a1b3b5 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_clock.c +++ b/tools/testing/selftests/kvm/x86/hyperv_clock.c
@@ -248,7 +248,7 @@ int main(void) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); /* Reset kvmclock triggering TSC page update */ if (stage == 7 || stage == 8 || stage == 10) {
diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 63ea153..c81f2dd 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
@@ -23,7 +23,11 @@ static int ud_count; static void guest_ud_handler(struct ex_regs *regs) { ud_count++; - regs->rip += 3; /* VMLAUNCH */ + /* + * VMLAUNCH insn can't be easily covered by KVM_ASM_SAFE framework but + * luckily the instruction is always three bytes. + */ + regs->rip += 3; } static void guest_nmi_handler(struct ex_regs *regs) @@ -89,24 +93,23 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); evmcs_enable(); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); - GUEST_ASSERT(load_evmcs(hv_pages)); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); + load_evmcs(hv_pages); + /* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */ + GUEST_ASSERT(vmptrst() == -1); GUEST_SYNC(4); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); GUEST_SYNC(5); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); current_evmcs->revision_id = -1u; - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); current_evmcs->revision_id = EVMCS_VERSION; GUEST_SYNC(6); - vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmreadz(PIN_BASED_VM_EXEC_CONTROL) | + vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_NMI_EXITING); /* L2 TLB flush setup */ @@ -117,71 +120,79 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, current_vp_assist->nested_control.features.directhypercall = 1; *(u32 *)(hv_pages->partition_assist) = 0; - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); + vmlaunch(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); + GUEST_ASSERT(vmptrst() == hv_pages->enlightened_vmcs_gpa); /* * NMI forces L2->L1 exit, resuming L2 and hope that EVMCS is * up-to-date (RIP points where it should and not at the beginning * of l2_guest_code(). GUEST_SYNC(9) checks that. */ - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_SYNC(10); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Intercept RDMSR 0xc0000100 */ - vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | + vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); __set_bit(MSR_FS_BASE & 0x1fff, vmx_pages->msr + 0x400); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable enlightened MSR bitmap */ current_evmcs->hv_enlightenments_control.msr_bitmap = 1; - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Intercept RDMSR 0xc0000101 without telling KVM about it */ __set_bit(MSR_GS_BASE & 0x1fff, vmx_pages->msr + 0x400); /* Make sure HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP is set */ current_evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; - GUEST_ASSERT(!vmresume()); + vmresume(); /* Make sure we don't see EXIT_REASON_MSR_READ here so eMSR bitmap works */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Now tell KVM we've changed MSR-Bitmap */ current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* * L2 TLB flush test. First VMCALL should be handled directly by L0, * no VMCALL exit expected. */ - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable synthetic vmexit */ *(u32 *)(hv_pages->partition_assist) = 1; - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(11); - /* Try enlightened vmptrld with an incorrect GPA */ + /* VMPTRLD instruction causes #UD after enlightened VMLAUNCH */ + GUEST_ASSERT(__vmptrld(hv_pages->enlightened_vmcs_gpa) == UD_VECTOR); + + /* + * Try enlightened vmptrld with an incorrect GPA. GUEST_SYNC(12) signals + * the host to enable guest_ud_handler() which cannot be enabled beforehand + * to not override the default fixup handler from KVM_ASM_SAFE(). + */ + GUEST_SYNC(12); evmcs_vmptrld(0xdeadbeef, hv_pages->enlightened_vmcs); - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); GUEST_ASSERT(ud_count == 1); GUEST_DONE(); } @@ -221,7 +232,7 @@ static struct kvm_vcpu *save_restore_vm(struct kvm_vm *vm, vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); return vcpu; } @@ -253,7 +264,6 @@ int main(int argc, char *argv[]) vcpu_args_set(vcpu, 3, vmx_pages_gva, hv_pages_gva, addr_gva2gpa(vm, hcall_page)); vcpu_set_msr(vcpu, HV_X64_MSR_VP_INDEX, vcpu->id); - vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); pr_info("Running L1 which uses EVMCS to run L2\n"); @@ -277,13 +287,13 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); vcpu = save_restore_vm(vm, vcpu); /* Force immediate L2->L1 exit before resuming */ if (stage == 8) { - pr_info("Injecting NMI into L1 before L2 had a chance to run after restore\n"); + pr_debug("Injecting NMI into L1 before L2 had a chance to run after restore\n"); inject_nmi(vcpu); } @@ -293,9 +303,14 @@ int main(int argc, char *argv[]) * KVM_STATE_NESTED_EVMCS is not lost. */ if (stage == 9) { - pr_info("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); + pr_debug("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); vcpu = save_restore_vm(vm, vcpu); } + + if (stage == 12) { + pr_debug("Trying enlightened VMLAUNCH with an invalid PTR\n"); + vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); + } } done:
diff --git a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c index 1f74b0f..3a3dc3107 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c +++ b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
@@ -186,7 +186,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); }
diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 25a182b..e1d2730 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
@@ -42,13 +42,13 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmlaunch()); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmlaunch(); } static void guest_code(void *test_data)
diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c index b974cfb..1f4a5e4 100644 --- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c +++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
@@ -30,13 +30,13 @@ static void l2_guest_code(void) static void l1_vmx_code(struct vmx_pages *vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(0); }
diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index 26b474b..7e5f9ae 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
@@ -93,8 +93,8 @@ void l1_vmx_code(struct vmx_pages *vmx) void *l2_rip; GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); if (vmx->eptp_gpa) l2_rip = l2_guest_code_tdp_enabled; @@ -104,9 +104,9 @@ void l1_vmx_code(struct vmx_pages *vmx) prepare_vmcs(vmx, l2_rip); GUEST_SYNC(TEST_SYNC_NO_FAULT); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_SYNC(TEST_SYNC_NO_FAULT); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); }
diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index e08c6b0..68d9ce9 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c
@@ -66,15 +66,15 @@ static void guest_code(void *test_data) BIT_ULL(INTERCEPT_HLT); vmcb->control.intercept_exceptions = 0; } else { - GUEST_ASSERT(prepare_for_vmx_operation(test_data)); - GUEST_ASSERT(load_vmcs(test_data)); + prepare_for_vmx_operation(test_data); + load_vmcs(test_data); prepare_vmcs(test_data, NULL); - GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0)); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, 0)); + vmwrite(GUEST_IDTR_LIMIT, 0); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmwrite(EXCEPTION_BITMAP, 0); - vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | + vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_PAUSE_EXITING | CPU_BASED_HLT_EXITING); } @@ -102,10 +102,13 @@ static void guest_code(void *test_data) exit_insn_len = vmcb->control.next_rip - vmcb->save.rip; GUEST_ASSERT_EQ(vmcb->save.rip, (u64)l2_instruction); } else { - GUEST_ASSERT_EQ(i ? vmresume() : vmlaunch(), 0); - exit_reason = vmreadz(VM_EXIT_REASON); - exit_insn_len = vmreadz(VM_EXIT_INSTRUCTION_LEN); - GUEST_ASSERT_EQ(vmreadz(GUEST_RIP), (u64)l2_instruction); + if (!i) + vmlaunch(); + else + vmresume(); + exit_reason = vmread(VM_EXIT_REASON); + exit_insn_len = vmread(VM_EXIT_INSTRUCTION_LEN); + GUEST_ASSERT_EQ(vmread(GUEST_RIP), (u64)l2_instruction); } __GUEST_ASSERT(exit_reason == insn->exit_reason[f],
diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index aeec312..f474325 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
@@ -38,6 +38,7 @@ */ #define GP_ERROR_CODE_AMD ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) #define GP_ERROR_CODE_INTEL ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG | ERROR_CODE_EXT_FLAG) +#define GP_ERROR_CODE_ZHAOXIN ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) /* * Intel and AMD both shove '0' into the error code on #DF, regardless of what @@ -110,43 +111,47 @@ static void l1_svm_code(struct svm_test_data *svm) static void vmx_run_l2(void *l2_code, int vector, u32 error_code) { - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code)); + vmwrite(GUEST_RIP, (u64)l2_code); - GUEST_ASSERT_EQ(vector == SS_VECTOR ? vmlaunch() : vmresume(), 0); + if (vector == SS_VECTOR) + vmlaunch(); + else + vmresume(); if (vector == FAKE_TRIPLE_FAULT_VECTOR) return; - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), vector); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_ERROR_CODE), error_code); - GUEST_ASSERT(!vmreadz(GUEST_INTERRUPTIBILITY_INFO)); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), vector); + GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_ERROR_CODE), error_code); + GUEST_ASSERT(!vmread(GUEST_INTERRUPTIBILITY_INFO)); } static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); - GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0); + vmwrite(GUEST_IDTR_LIMIT, 0); /* * VMX disallows injecting an exception with error_code[31:16] != 0, * and hardware will never generate a VM-Exit with bits 31:16 set. * KVM should likewise truncate the "bad" userspace value. */ - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF); vmx_run_l2(l2_ss_pending_test, SS_VECTOR, (u16)SS_ERROR_CODE); - vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_INTEL); + vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, + host_cpu_is_zhaoxin ? GP_ERROR_CODE_ZHAOXIN : GP_ERROR_CODE_INTEL); - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF); vmx_run_l2(l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE); - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS); vmx_run_l2(l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); }
diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 8c2ba96..6e14782 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
@@ -40,22 +40,22 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) { uintptr_t save_cr3; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); /* Try to run L2 with invalid CR3 and make sure it fails */ - save_cr3 = vmreadz(GUEST_CR3); + save_cr3 = vmread(GUEST_CR3); vmwrite(GUEST_CR3, -1ull); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == (EXIT_REASON_FAILED_VMENTRY | EXIT_REASON_INVALID_STATE)); /* Now restore CR3 and make sure L2 runs successfully */ vmwrite(GUEST_CR3, save_cr3); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); }
diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index 2e04563..fae77d8 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c
@@ -55,25 +55,25 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, u64 exit_qual; GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, l2_entry); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* Verify we got an EPT violation exit */ - __GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, + __GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, "Expected EPT violation (0x%x), got 0x%lx", EXIT_REASON_EPT_VIOLATION, - vmreadz(VM_EXIT_REASON)); + vmread(VM_EXIT_REASON)); - __GUEST_ASSERT(vmreadz(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, + __GUEST_ASSERT(vmread(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, "Expected guest_physical_address = 0x%lx, got 0x%lx", expected_fault_gpa, - vmreadz(GUEST_PHYSICAL_ADDRESS)); + vmread(GUEST_PHYSICAL_ADDRESS)); - exit_qual = vmreadz(EXIT_QUALIFICATION); + exit_qual = vmread(EXIT_QUALIFICATION); /* * Note, EPT page table accesses are always read+write, e.g. so that
diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index cb79d7b..6a1a88e 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
@@ -86,17 +86,17 @@ static void l1_guest_code(void *data) struct vmx_pages *vmx_pages = data; u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); } else { struct svm_test_data *svm = data;
diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 18f7658..bfb1ff4 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
@@ -1,14 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * vmx_nested_tsc_scaling_test - * * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * This test case verifies that nested TSC scaling behaves as expected when - * both L1 and L2 are scaled using different ratios. For this test we scale - * L1 down and scale L2 up. + * both L1 and L2 are scaled using different ratios. */ - +#include <linux/math64.h> #include <time.h> #include "kvm_util.h" @@ -16,16 +13,11 @@ #include "svm_util.h" #include "kselftest.h" -/* L2 is scaled up (from L1's perspective) by this factor */ -#define L2_SCALE_FACTOR 4ULL +/* L2's TSC multiplier and offset, relative to L1. */ +static u64 l2_multiplier; +static u64 l2_offset; -#define TSC_OFFSET_L2 ((u64)-33125236320908) -#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48) - -enum { USLEEP, UCHECK_L1, UCHECK_L2 }; -#define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) -#define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq) - +enum ucall_cmd { UCALL_SLEEP, UCALL_CHECK_L1, UCALL_CHECK_L2 }; /* * This function checks whether the "actual" TSC frequency of a guest matches @@ -33,25 +25,20 @@ enum { USLEEP, UCHECK_L1, UCHECK_L2 }; * measurements, a difference of 1% between the actual and the expected value * is tolerated. */ -static void compare_tsc_freq(u64 actual, u64 expected) +static void host_check_tsc_freq(int level, u64 actual, u64 expected) { u64 tolerance, thresh_low, thresh_high; - tolerance = expected / 100; + tolerance = max(expected / 100, (u64)1); thresh_low = expected - tolerance; thresh_high = expected + tolerance; - TEST_ASSERT(thresh_low < actual, - "TSC freq is expected to be between %"PRIu64" and %"PRIu64 - " but it actually is %"PRIu64, - thresh_low, thresh_high, actual); - TEST_ASSERT(thresh_high > actual, - "TSC freq is expected to be between %"PRIu64" and %"PRIu64 - " but it actually is %"PRIu64, - thresh_low, thresh_high, actual); + TEST_ASSERT(thresh_low <= actual && thresh_high >= actual, + "L%u TSC freq is '%lu', expected to be between %lu and %lu", + level, actual, thresh_low, thresh_high); } -static void check_tsc_freq(int level) +static void guest_check_tsc_freq(enum ucall_cmd check_level) { u64 tsc_start, tsc_end, tsc_freq; @@ -62,17 +49,17 @@ static void check_tsc_freq(int level) * be good enough for the purposes of this test. */ tsc_start = rdmsr(MSR_IA32_TSC); - GUEST_SLEEP(1); + GUEST_SYNC2(UCALL_SLEEP, 1); tsc_end = rdmsr(MSR_IA32_TSC); tsc_freq = tsc_end - tsc_start; - GUEST_CHECK(level, tsc_freq); + GUEST_SYNC2(check_level, tsc_freq); } static void l2_guest_code(void) { - check_tsc_freq(UCHECK_L2); + guest_check_tsc_freq(UCALL_CHECK_L2); /* exit to L1 */ __asm__ __volatile__("vmcall"); @@ -81,19 +68,20 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { /* check that L1's frequency looks alright before launching L2 */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); generic_svm_setup(svm, l2_guest_code); + svm->vmcb->control.tsc_offset = l2_offset; /* enable TSC scaling for L2 */ - wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32); + wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier); /* launch L2 */ run_guest(svm->vmcb, svm->vmcb_gpa); GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL); /* check that L1's frequency still looks good */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); GUEST_DONE(); } @@ -103,33 +91,32 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) u32 control; /* check that L1's frequency looks alright before launching L2 */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* prepare the VMCS for L2 execution */ prepare_vmcs(vmx_pages, l2_guest_code); /* enable TSC offsetting and TSC scaling for L2 */ - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_TSC_SCALING; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - vmwrite(TSC_OFFSET, TSC_OFFSET_L2); - vmwrite(TSC_MULTIPLIER, TSC_MULTIPLIER_L2); - vmwrite(TSC_MULTIPLIER_HIGH, TSC_MULTIPLIER_L2 >> 32); + vmwrite(TSC_OFFSET, l2_offset); + vmwrite(TSC_MULTIPLIER, l2_multiplier); /* launch L2 */ - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* check that L1's frequency still looks good */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); GUEST_DONE(); } @@ -142,44 +129,24 @@ static void l1_guest_code(void *data) l1_svm_code(data); } -int main(int argc, char *argv[]) +static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq, + u64 __l2_multiplier) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; - gva_t guest_gva = 0; + gva_t guest_gva; - u64 tsc_start, tsc_end; - u64 tsc_khz; - u64 l1_scale_factor; - u64 l0_tsc_freq = 0; - u64 l1_tsc_freq = 0; - u64 l2_tsc_freq = 0; - - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || - kvm_cpu_has(X86_FEATURE_SVM)); - TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); - TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); - - /* - * We set L1's scale factor to be a random number from 2 to 10. - * Ideally we would do the same for L2's factor but that one is - * referenced by both main() and l1_guest_code() and using a global - * variable does not work. - */ - srand(time(NULL)); - l1_scale_factor = (rand() % 9) + 2; - printf("L1's scale down factor is: %"PRIu64"\n", l1_scale_factor); - printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR); - - tsc_start = rdtsc(); - sleep(1); - tsc_end = rdtsc(); - - l0_tsc_freq = tsc_end - tsc_start; - printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq); + printf("Testing L0 freq = %lu, L1 freq = %lu, L2 freq = %lu, L2 mult = 0x%lx\n", + l0_tsc_freq, l1_tsc_freq, l2_tsc_freq, __l2_multiplier); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + l2_offset = kvm_random_u64(&kvm_rng); + sync_global_to_guest(vm, l2_offset); + + l2_multiplier = __l2_multiplier; + sync_global_to_guest(vm, l2_multiplier); + if (kvm_cpu_has(X86_FEATURE_VMX)) vcpu_alloc_vmx(vm, &guest_gva); else @@ -187,11 +154,7 @@ int main(int argc, char *argv[]) vcpu_args_set(vcpu, 1, guest_gva); - tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL); - TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed"); - - /* scale down L1's TSC frequency */ - vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (tsc_khz / l1_scale_factor)); + vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)(l1_tsc_freq / 1000)); for (;;) { struct ucall uc; @@ -204,24 +167,16 @@ int main(int argc, char *argv[]) REPORT_GUEST_ASSERT(uc); case UCALL_SYNC: switch (uc.args[0]) { - case USLEEP: + case UCALL_SLEEP: sleep(uc.args[1]); break; - case UCHECK_L1: - l1_tsc_freq = uc.args[1]; - printf("L1's TSC frequency is around: %"PRIu64 - "\n", l1_tsc_freq); - - compare_tsc_freq(l1_tsc_freq, - l0_tsc_freq / l1_scale_factor); + case UCALL_CHECK_L1: + printf("L1's observed TSC frequency: %lu\n", uc.args[1]); + host_check_tsc_freq(1, uc.args[1], l1_tsc_freq); break; - case UCHECK_L2: - l2_tsc_freq = uc.args[1]; - printf("L2's TSC frequency is around: %"PRIu64 - "\n", l2_tsc_freq); - - compare_tsc_freq(l2_tsc_freq, - l1_tsc_freq * L2_SCALE_FACTOR); + case UCALL_CHECK_L2: + printf("L2's observed TSC frequency: %lu\n", uc.args[1]); + host_check_tsc_freq(2, uc.args[1], l2_tsc_freq); break; } break; @@ -234,5 +189,108 @@ int main(int argc, char *argv[]) done: kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + u64 min_freq, min_multiplier, max_freq, max_multiplier, l1_max_freq, l1_min_freq; + u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale; + u8 frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32; + struct kvm_vm *vm; + + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || + kvm_cpu_has(X86_FEATURE_SVM)); + TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); + TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); + + /* + * Create a dummy VM to get KVM's default TSC frequency. All CPUs that + * support TSC scaling should have a constant TSC, i.e. there's no need + * to calibrate the "real" TSC. But do sanity check that the observed + * TSC is within range of KVM's reported TSC frequency. + */ + vm = vm_create_barebones(); + l0_tsc_freq = (u64)__vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL) * 1000; + TEST_ASSERT(l0_tsc_freq, "vcpu ioctl KVM_GET_TSC_KHZ failed"); + kvm_vm_free(vm); + + printf("L0 TSC frequency is: %lu\n", l0_tsc_freq); + + tsc_start = rdtsc(); + sleep(1); + tsc_end = rdtsc(); + host_check_tsc_freq(0, tsc_end - tsc_start, l0_tsc_freq); + + /* Sanity check the frequency reported by KVM_GET_TSC_KHZ. */ + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq, l0_tsc_freq, BIT_ULL(frac_bits)); + + /* + * Scale L1 and L2 up and down at random factors from 2 to 10. Current + * CPUs have two full orders of magnitude of "breathing room" before an + * ultrafast L0 TSC frequency multiplied by 10x will encounter KVM's + * signed 32-bit limit on L1's frequency, and it's highly unlikely a + * CPU that supports TSC scaling will show up running at 100MHz, i.e. + * underflowing KVM's minimum 1KHz frequency is extremely unlikely. + * + * For L2, the frequency is limited only by what hardware can support, + * not by KVM's limits. SVM provides 8 bits of integer scale up, and + * 32 bits of fractional scale down, i.e. can scale up 255x and down a + * comical amount, so scaling up 100x and down 1/100 is well within + * hardware's capabilities (VMX provides 16 bits of "up" and 48 bits of + * "down"). + */ + l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; + l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale, + mul_u64_u64_div64(l0_tsc_freq, l2_scale, l1_scale), + (l2_scale << frac_bits)); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale, + mul_u64_u64_div64(l0_tsc_freq, l1_scale, l2_scale), + (1ull << frac_bits) / l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale, + l0_tsc_freq / l1_scale / l2_scale, + (1ull << frac_bits) / l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale, + l0_tsc_freq * l1_scale * l2_scale, + (l2_scale << frac_bits)); + + /* + * Test that KVM saturates L2's frequency on both ends if the resulting + * L2 TSC frequency would be below or above what hardware can support. + * Because L2 = L0 * (L1_mult >> frac) * (L2_mult >> frac) needs to be + * distilled down to a single multiplier, very small/large multipliers + * will underflow/overflow the minimum/maximum multiplier supported by + * hardware when L1 and L2 multipliers are combined. KVM's behavior is + * saturate on {under,over}flow, i.e. to run at the min/max frequency. + * + * Note, userspace can only program L1's frequency in KHz, i.e. can't + * specify an exact multiplier. As a result, the minimum and maximum + * frequencies are different for L1 vs L2, because L1 is constrained by + * hardware *and* KVM, whereas L2 is constrained only by hardware. + */ + min_multiplier = 1; + min_freq = mul_u64_u64_div64(l0_tsc_freq, min_multiplier, BIT_ULL(frac_bits)); + min_freq = max(min_freq, (u64)1); + l1_min_freq = max(min_freq, (u64)1 * 1000); + test_tsc_scaling(l0_tsc_freq, l1_min_freq, min_freq, 1); + + /* + * SVM takes a 40-bit value (right shifted by 32), while VMX takes a + * 64-bit value (right shifted by 48). mul_u64_u64_shr() isn't (yet) + * available in selftests, but mul_u64_u64_div64() does nicely since, + * albeit more slowly (performance is obviously not a concern). Note, + * because KVM_GET_TSC_KHZ returns a signed 32-bit integer, KVM limits + * KVM_SET_TSC_KHZ to INT_MAX, even though hardware (both SVM and VMX) + * supports much higher frequencies. + */ + max_multiplier = kvm_cpu_has(X86_FEATURE_VMX) ? -1ull : GENMASK_U64(39, 0); + max_freq = mul_u64_u64_div64(l0_tsc_freq, max_multiplier, BIT_ULL(frac_bits)); + l1_max_freq = min(max_freq, (u64)INT32_MAX * 1000); + test_tsc_scaling(l0_tsc_freq, l1_max_freq, max_freq, max_multiplier); + return 0; }
diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index ce204ce..2ae698c 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c
@@ -61,57 +61,55 @@ static void l1_vmx_code(struct vmx_pages *vmx, struct hyperv_test_pages *hv_page evmcs_enable(); } - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); if (hv_pages) { - GUEST_ASSERT(load_evmcs(hv_pages)); + load_evmcs(hv_pages); current_evmcs->hv_enlightenments_control.msr_bitmap = 1; } else { - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); } prepare_vmcs(vmx, NULL); - GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0); + vmwrite(GUEST_RIP, (unsigned long)l2_guest_code); - control = vmreadz(PIN_BASED_VM_EXEC_CONTROL); + control = vmread(PIN_BASED_VM_EXEC_CONTROL); control |= PIN_BASED_EXT_INTR_MASK; vmwrite(PIN_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_TPR_SHADOW; - GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY; control &= (rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32); - GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); - GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, - vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); + vmlaunch(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); } static void l1_vmx_code_part2(void) { u64 control; - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control &= ~CPU_BASED_TPR_SHADOW; - GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control &= ~(SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); - GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); - GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, - vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); + vmresume(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); } static void l1_test_x2apic_intercepts(void)
diff --git a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c index 7095006..e3b2003 100644 --- a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c +++ b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c
@@ -120,7 +120,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, } else { TEST_ASSERT(r == -1 && errno == EPERM, "This process should not have permission to disable NX huge pages"); - return; + goto done; } } @@ -213,6 +213,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, check_2m_page_count(vm, disable_nx_huge_pages ? 3 : 2); check_split_count(vm, 0); +done: kvm_vm_free(vm); }
diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c index 3aa0673..33b6103 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c
@@ -202,15 +202,18 @@ static void guest_test_explicit_conversion(u64 base_gpa, bool do_fallocate) guest_sync_shared(gpa, size, p3, p4); memcmp_g(gpa, p4, size); - /* Reset the shared memory back to the initial pattern. */ - memset((void *)gpa, init_p, size); - /* * Free (via PUNCH_HOLE) *all* private memory so that the next * iteration starts from a clean slate, e.g. with respect to * whether or not there are pages/folios in guest_mem. */ guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true); + + /* + * Hole-punching above zeroed private memory. Reset shared + * memory in preparation for the next GUEST_STAGE. + */ + memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE); } } @@ -303,9 +306,12 @@ static void handle_exit_hypercall(struct kvm_vcpu *vcpu) if (do_fallocate) vm_guest_mem_fallocate(vm, gpa, size, map_shared); - if (set_attributes) - vm_set_memory_attributes(vm, gpa, size, - map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE); + if (set_attributes) { + u64 attrs = map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; + + vm_mem_set_memory_attributes(vm, gpa, size, attrs); + } + run->hypercall.ret = 0; } @@ -349,8 +355,21 @@ static void *__test_mem_conversions(void *__vcpu) size_t nr_bytes = min_t(size_t, vm->page_size, size - i); u8 *hva = addr_gpa2hva(vm, gpa + i); - /* In all cases, the host should observe the shared data. */ - memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + if (kvm_has_gmem_attributes && + uc.args[0] == SYNC_PRIVATE) { + TEST_EXPECT_SIGBUS(READ_ONCE(*hva)); + } else { + /* + * If not testing in-place conversion + * (dual backing), the host should + * always observe shared data, since the + * shared memory is a separate page. + * + * For SYNC_SHARED, test that the host + * can see shared memory. + */ + memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + } /* For shared, write the new pattern to guest memory. */ if (uc.args[0] == SYNC_SHARED) @@ -366,20 +385,29 @@ static void *__test_mem_conversions(void *__vcpu) } } +/* Align each vCPU's chunk of memory naturally to the size of the backing store. */ +static size_t compute_per_cpu_size(enum vm_mem_backing_src_type src_type) +{ + size_t alignment; + + if (kvm_has_gmem_attributes) + alignment = getpagesize(); + else + alignment = get_backing_src_pagesz(src_type); + + return align_up(PER_CPU_DATA_SIZE, max_t(size_t, SZ_2M, alignment)); +} + static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_vcpus, u32 nr_memslots) { - /* - * Allocate enough memory so that each vCPU's chunk of memory can be - * naturally aligned with respect to the size of the backing store. - */ - const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type)); - const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); + const size_t per_cpu_size = compute_per_cpu_size(src_type); const size_t memfd_size = per_cpu_size * nr_vcpus; const size_t slot_size = memfd_size / nr_memslots; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; pthread_t threads[KVM_MAX_VCPUS]; struct kvm_vm *vm; + u64 gmem_flags; int memfd, i; const struct vm_shape shape = { @@ -394,12 +422,16 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_v vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE)); - memfd = vm_create_guest_memfd(vm, memfd_size, 0); + gmem_flags = 0; + if (kvm_has_gmem_attributes) + gmem_flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; + + memfd = vm_create_guest_memfd(vm, memfd_size, gmem_flags); for (i = 0; i < nr_memslots; i++) vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, BASE_DATA_SLOT + i, slot_size / vm->page_size, - KVM_MEM_GUEST_MEMFD, memfd, slot_size * i); + KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, gmem_flags); for (i = 0; i < nr_vcpus; i++) { gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size; @@ -459,6 +491,8 @@ int main(int argc, char *argv[]) while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) { switch (opt) { case 's': + TEST_ASSERT(!kvm_has_gmem_attributes, + "src_type is only configurable when testing without in-place conversion"); src_type = parse_backing_src_type(optarg); break; case 'n':
diff --git a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c index e6d16e9..f7228e0f 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c
@@ -62,8 +62,9 @@ static void test_private_access_memslot_deleted(void) virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES); - /* Request to access page privately */ - vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); + /* Request to access page privately. */ + if (!kvm_has_gmem_attributes) + vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); kvm_pthread_create(&vm_thread, NULL, (pthread_fn_t)run_vcpu_get_exit_reason, (void *)vcpu); @@ -73,10 +74,26 @@ static void test_private_access_memslot_deleted(void) kvm_pthread_join(vm_thread, &thread_return); exit_reason = (u32)(u64)thread_return; - TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); - TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); - TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); - TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + /* + * If attributes are tracked per-gmem, deleting the memslot that points + * at the gmem instance effectively makes the memory shared, and so the + * read should trigger emulated MMIO. + * + * If attributes are tracked per-VM, deleting the memslot shouldn't + * affect the private attribute, and so KVM should generate a memory + * fault exit (emulated MMIO on private GPAs is disallowed). + */ + if (kvm_has_gmem_attributes) { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MMIO); + TEST_ASSERT_EQ(vcpu->run->mmio.phys_addr, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->mmio.len, sizeof(u64)); + TEST_ASSERT_EQ(vcpu->run->mmio.is_write, false); + } else { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); + TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); + TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + } kvm_vm_free(vm); } @@ -87,6 +104,13 @@ static void test_private_access_memslot_not_private(void) struct kvm_vcpu *vcpu; u32 exit_reason; + /* + * Accessing non-private memory as private with a software-protected VM + * isn't possible when doing in-place conversions. + */ + if (kvm_has_gmem_attributes) + return; + vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu, guest_repeatedly_read);
diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index 507391a..2405692 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c
@@ -98,17 +98,17 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, guest_access_memory); - GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR))); + vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); while (1) { - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); - GUEST_ASSERT(!vmresume()); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); + vmresume(); } }
diff --git a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c index 8e38986..909cd9e 100644 --- a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c +++ b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c
@@ -69,7 +69,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage + 1, "Stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); test_set_bsp_busy(vcpu, "while running vm"); break; case UCALL_DONE:
diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c index 603226f..5afc418 100644 --- a/tools/testing/selftests/kvm/x86/set_sregs_test.c +++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c
@@ -135,6 +135,17 @@ static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) /* NW without CD is illegal, as is PG without PE. */ TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_NW); TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_PG); + + /* CR8 bits 3:0 are writable; bits 63:4 are reserved. */ + vcpu_sregs_get(vcpu, &sregs); + sregs.cr8 = 0xf; + vcpu_sregs_set(vcpu, &sregs); + + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT_EQ(sregs.cr8, 0xf); + + for (i = 4; i < 64; i++) + TEST_INVALID_SREG_BIT(vcpu, cr8, sregs, BIT_ULL(i)); } static void test_efer_bits(struct kvm_vcpu *vcpu, u64 efer)
diff --git a/tools/testing/selftests/kvm/x86/sev_dbg_test.c b/tools/testing/selftests/kvm/x86/sev_dbg_test.c index eaa8201..0277e38 100644 --- a/tools/testing/selftests/kvm/x86/sev_dbg_test.c +++ b/tools/testing/selftests/kvm/x86/sev_dbg_test.c
@@ -14,9 +14,11 @@ static u8 *data; static u8 src[BUFFER_SIZE] __aligned(PAGE_SIZE); static u8 dst[BUFFER_SIZE] __aligned(PAGE_SIZE); -static void validate_dst(int i, int nr_bytes, u8 pattern) +static void validate_dst(int offset, int nr_bytes, u8 pattern) { - for ( ; i < nr_bytes; i++) + int i; + + for (i = offset; i < offset + nr_bytes; i++) TEST_ASSERT(dst[i] == pattern, "Expected 0x%x at byte %u, got 0x%x", pattern, i, dst[i]);
diff --git a/tools/testing/selftests/kvm/x86/sev_smoke_test.c b/tools/testing/selftests/kvm/x86/sev_smoke_test.c index bf27b61..19f76f7 100644 --- a/tools/testing/selftests/kvm/x86/sev_smoke_test.c +++ b/tools/testing/selftests/kvm/x86/sev_smoke_test.c
@@ -178,7 +178,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy) case UCALL_SYNC: continue; case UCALL_DONE: - return; + goto done; case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); default: @@ -187,6 +187,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy) } } +done: kvm_vm_free(vm); }
diff --git a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c index 3dca85e..8d182271 100644 --- a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c
@@ -62,11 +62,12 @@ int main(int argc, char *argv[]) TEST_ASSERT(rc, "KVM_CAP_EXIT_ON_EMULATION_FAILURE is unavailable"); vm_enable_cap(vm, KVM_CAP_EXIT_ON_EMULATION_FAILURE, 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, - MEM_REGION_GPA, MEM_REGION_SLOT, - MEM_REGION_SIZE / PAGE_SIZE, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / PAGE_SIZE); + gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE, - MEM_REGION_GPA, MEM_REGION_SLOT); + MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc"); virt_map(vm, MEM_REGION_GVA, MEM_REGION_GPA, 1); hva = addr_gpa2hva(vm, MEM_REGION_GPA);
diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c index d1edafd..377df09 100644 --- a/tools/testing/selftests/kvm/x86/smm_test.c +++ b/tools/testing/selftests/kvm/x86/smm_test.c
@@ -78,8 +78,8 @@ static void guest_code(void *arg) if (this_cpu_has(X86_FEATURE_SVM)) { generic_svm_setup(svm, l2_guest_code); } else { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); }
diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 4a1056a..500b097 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c
@@ -59,13 +59,13 @@ void vmx_l2_guest_code(void) vmcall(); /* L1 has now set up a shadow VMCS for us. */ - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); GUEST_SYNC(10); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); - GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0fffee)); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); + vmwrite(GUEST_RIP, 0xc0fffee); GUEST_SYNC(11); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0fffee); - GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0ffffee)); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0fffee); + vmwrite(GUEST_RIP, 0xc0ffffee); GUEST_SYNC(12); /* Done, exit to L1 and never come back. */ @@ -75,61 +75,61 @@ void vmx_l2_guest_code(void) static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) { GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); - GUEST_ASSERT(load_vmcs(vmx_pages)); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + load_vmcs(vmx_pages); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_SYNC(4); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, vmx_l2_guest_code); GUEST_SYNC(5); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); + vmlaunch(); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* Check that the launched state is preserved. */ - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(7); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + 3); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + 3); vmwrite(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa); - GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); - GUEST_ASSERT(vmlaunch()); + vmptrld(vmx_pages->shadow_vmcs_gpa); + GUEST_ASSERT(__vmlaunch()); GUEST_SYNC(8); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); vmwrite(GUEST_RIP, 0xc0ffee); GUEST_SYNC(9); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); - GUEST_ASSERT(!vmptrld(vmx_pages->vmcs_gpa)); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmptrld(vmx_pages->vmcs_gpa); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + vmptrld(vmx_pages->shadow_vmcs_gpa); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); GUEST_SYNC(13); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); } static void __attribute__((__flatten__)) guest_code(void *arg) @@ -296,7 +296,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); state = vcpu_save_state(vcpu); memset(®s1, 0, sizeof(regs1)); @@ -343,7 +343,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } done:
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c deleted file mode 100644 index 6bc3012..0000000 --- a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c +++ /dev/null
@@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2026, Google LLC. - */ -#include "kvm_util.h" -#include "vmx.h" -#include "svm_util.h" -#include "kselftest.h" - - -static void l2_guest_code(void) -{ - unsigned long efer = rdmsr(MSR_EFER); - - /* generic_svm_setup() initializes EFER_SVME set for L2 */ - GUEST_ASSERT(efer & EFER_SVME); - wrmsr(MSR_EFER, efer & ~EFER_SVME); - - /* Unreachable, L1 should be shutdown */ - GUEST_ASSERT(0); -} - -static void l1_guest_code(struct svm_test_data *svm) -{ - generic_svm_setup(svm, l2_guest_code); - run_guest(svm->vmcb, svm->vmcb_gpa); - - /* Unreachable, L1 should be shutdown */ - GUEST_ASSERT(0); -} - -int main(int argc, char *argv[]) -{ - struct kvm_vcpu *vcpu; - struct kvm_vm *vm; - gva_t nested_gva = 0; - - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); - - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); - - vcpu_alloc_svm(vm, &nested_gva); - vcpu_args_set(vcpu, 1, nested_gva); - - vcpu_run(vcpu); - TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); - - kvm_vm_free(vm); - return 0; -}
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c new file mode 100644 index 0000000..fa0d76a --- /dev/null +++ b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c
@@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Tests for KVM's handling of EFER bits whose behavior is tied to nested SVM. + * + * Copyright (C) 2026, Google LLC. + */ +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "svm_util.h" +#include "kselftest.h" + +static bool l2_ran; + +static void l2_clear_efer_svme(void) +{ + u64 efer = rdmsr(MSR_EFER); + + /* generic_svm_setup() initializes EFER_SVME set for L2 */ + GUEST_ASSERT(efer & EFER_SVME); + wrmsr(MSR_EFER, efer & ~EFER_SVME); + + /* Unreachable, L1 should be shutdown */ + GUEST_ASSERT(0); +} + +static void l1_clear_efer_svme(struct svm_test_data *svm) +{ + generic_svm_setup(svm, l2_clear_efer_svme); + run_guest(svm->vmcb, svm->vmcb_gpa); + + /* Unreachable, L1 should be shutdown */ + GUEST_ASSERT(0); +} + +static void l2_lmsle(void) +{ + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + l2_ran = true; + vmmcall(); +} + +static void l1_lmsle(struct svm_test_data *svm) +{ + bool lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + struct vmcb *vmcb = svm->vmcb; + u64 efer = rdmsr(MSR_EFER); + + /* + * Selftests' vCPUs are created with EFER.LMSLE clear; the sub-tests + * below need to start from a clean slate. + */ + GUEST_ASSERT(!(efer & EFER_LMSLE)); + GUEST_ASSERT(!l2_ran); + + /* + * Per the APM, if EFER_LMSLE_MBZ is enumerated in CPUID, "64-bit mode + * segment limit checking is not supported and attempting to set + * EFER.LMSLE = 1 causes a #GP exception". + */ + if (lmsle_mbz) { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), GP_VECTOR); + GUEST_ASSERT(!(rdmsr(MSR_EFER) & EFER_LMSLE)); + } else { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), 0); + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + + /* + * Restore EFER so that generic_svm_setup() doesn't propagate + * EFER.LMSLE into vmcb12 on its own, i.e. so that the VMRUN + * sub-test actually tests what it thinks it's testing. + */ + wrmsr(MSR_EFER, efer); + } + + /* + * VMRUN's consistency checks reject "any MBZ bit of EFER", i.e. a + * vmcb12 with EFER.LMSLE set must generate VMEXIT_INVALID when the + * defeature is enumerated. + */ + generic_svm_setup(svm, l2_lmsle); + vmcb->save.efer |= EFER_LMSLE; + run_guest(vmcb, svm->vmcb_gpa); + + if (lmsle_mbz) { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_ERR); + GUEST_ASSERT(!l2_ran); + } else { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL); + GUEST_ASSERT(l2_ran); + GUEST_ASSERT(vmcb->save.efer & EFER_LMSLE); + } + + GUEST_DONE(); +} + +static struct kvm_vcpu *create_l1_vcpu(struct kvm_vm **vm, void *l1_guest_code) +{ + struct kvm_vcpu *vcpu; + gva_t svm_gva; + + *vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + vcpu_alloc_svm(*vm, &svm_gva); + vcpu_args_set(vcpu, 1, svm_gva); + + return vcpu; +} + +static void test_enumeration(void) +{ + bool lmsle_mbz; + + /* + * EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], is a "defeature" bit, + * i.e. is set when the CPU does *not* support long mode segment + * limits. KVM enumerates the defeature if and only if KVM refuses to + * set EFER.LMSLE, i.e. if the CPU doesn't support LMSLE, or if KVM + * doesn't support nested SVM. Derive the expectation from raw CPUID + * and kvm_amd's "nested" module param rather than from + * kvm_cpu_has(X86_FEATURE_SVM), so that the assertion doesn't simply + * compare KVM's enumeration to itself. + */ + lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ) || + !this_cpu_has(X86_FEATURE_SVM) || + !kvm_is_nested_virtualization_enabled(); + + TEST_ASSERT_EQ(kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ), lmsle_mbz); + + ksft_test_result_pass("KVM enumerates EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_clear_efer_svme(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + vcpu = create_l1_vcpu(&vm, l1_clear_efer_svme); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); + + kvm_vm_free(vm); + ksft_test_result_pass("L2 clearing EFER.SVME shuts down L1\n"); +} + +static void test_lmsle(bool lmsle_mbz) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + vcpu = create_l1_vcpu(&vm, l1_lmsle); + + vcpu_set_or_clear_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ, + lmsle_mbz); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_DONE: + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } + + kvm_vm_free(vm); + ksft_test_result_pass("Guest EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_host_initiated_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 efer; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * EFER_LMSLE_MBZ is a guest CPUID consistency check, not a host + * capability, i.e. must not be enforced against host-initiated writes, + * so that userspace can set MSRs before it sets guest CPUID. + */ + efer = vcpu_get_msr(vcpu, MSR_EFER); + TEST_ASSERT(!(efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + vcpu_set_msr(vcpu, MSR_EFER, efer | EFER_LMSLE); + TEST_ASSERT_EQ(vcpu_get_msr(vcpu, MSR_EFER), efer | EFER_LMSLE); + + kvm_vm_free(vm); + ksft_test_result_pass("Host-initiated EFER.LMSLE=1 is allowed\n"); +} + +static void test_sregs_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_sregs sregs; + struct kvm_vm *vm; + int rc; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * Unlike KVM_SET_MSRS, KVM_SET_SREGS runs the full set of guest CPUID + * checks, i.e. rejects EFER.LMSLE even though it's host-initiated. + */ + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT(!(sregs.efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + sregs.efer |= EFER_LMSLE; + rc = _vcpu_sregs_set(vcpu, &sregs); + TEST_ASSERT(rc, "KVM allowed EFER.LMSLE with EFER_LMSLE_MBZ set"); + + kvm_vm_free(vm); + ksft_test_result_pass("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); +} + +int main(int argc, char *argv[]) +{ + bool has_nested_svm, has_lmsle; + + ksft_print_header(); + ksft_set_plan(6); + + test_enumeration(); + + /* + * The sub-tests below need to actually run a nested guest, and the + * EFER.LMSLE sub-tests additionally need KVM to allow EFER.LMSLE. + * It's KVM's view of the world, not raw CPUID, that dictates whether + * EFER.LMSLE is allowed, i.e. whether the defeature is emulated. + */ + has_nested_svm = kvm_cpu_has(X86_FEATURE_SVM); + has_lmsle = has_nested_svm && + !kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + + if (!has_nested_svm) + ksft_print_msg("Nested SVM unsupported\n"); + else if (!has_lmsle) + ksft_print_msg("KVM doesn't support EFER.LMSLE\n"); + + if (has_nested_svm) { + test_clear_efer_svme(); + test_lmsle(true); + } else { + ksft_test_result_skip("L2 clearing EFER.SVME shuts down L1\n"); + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=1\n"); + } + + if (has_lmsle) { + test_lmsle(false); + test_host_initiated_lmsle(); + test_sregs_lmsle(); + } else { + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=0\n"); + ksft_test_result_skip("Host-initiated EFER.LMSLE=1 is allowed\n"); + ksft_test_result_skip("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); + } + + ksft_finished(); +}
diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index 0d83516..b2b796f 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
@@ -25,14 +25,14 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) { GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 should triple fault after a triple fault event injected. */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); }
diff --git a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c index 9158396..5577977 100644 --- a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c +++ b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c
@@ -78,7 +78,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage) else ksft_test_result_fail( "stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); return; case UCALL_DONE: ksft_test_result_pass("stage %d passed\n", stage + 1);
diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index 463f73a..1d7a529 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
@@ -38,30 +38,30 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) { u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); vmwrite(APIC_ACCESS_ADDR, vmx_pages->apic_access_gpa); /* Try to launch L2 with the memory-backed APIC-access address. */ - GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_SYNC(vmread(APIC_ACCESS_ADDR)); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(APIC_ACCESS_ADDR, high_gpa); /* Try to resume L2 with the unbacked APIC-access address. */ - GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_SYNC(vmread(APIC_ACCESS_ADDR)); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); } @@ -77,7 +77,7 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX)); + TEST_REQUIRE(kvm_cpu_has_vmx_virtualize_apic_accesses()); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index f9b88a6..cda9e43 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
@@ -33,12 +33,12 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) { u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); @@ -62,9 +62,9 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * Run L2 to switch to x2APIC mode, which in turn will uninhibit APICv, * as KVM should force the APIC ID back to its default. */ - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); GUEST_ASSERT(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD); /* @@ -90,8 +90,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * handles the x2APIC => xAPIC transition and inhibits APICv while L2 * is active. */ - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_ASSERT(!(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD)); xapic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_DM_FIXED | GOOD_IPI_VECTOR);
diff --git a/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c index 2cae86d..08a0caa 100644 --- a/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c
@@ -109,7 +109,7 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; - TEST_REQUIRE(host_cpu_is_intel); + TEST_REQUIRE(host_cpu_is_intel_compatible); TEST_REQUIRE(!kvm_is_unrestricted_guest_enabled()); vm = vm_create_with_one_vcpu(&vcpu, guest_code);
diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index c837912..bbd1889 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
@@ -41,8 +41,8 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); @@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * library hasn't enabled it. Because KVM selftests jump directly to * 64-bit mode, unrestricted guest support isn't required. */ - GUEST_ASSERT(!(vmreadz(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || - !(vmreadz(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); + GUEST_ASSERT(!(vmread(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || + !(vmread(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 should triple fault after main() stuffs invalid guest state. */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); }
diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index 75073ef..7dcee11 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
@@ -37,8 +37,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(rdmsr(MSR_GS_BASE) == LA57_GS_BASE); GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); @@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) pml4_pa = pml5[0] & PHYSICAL_PAGE_MASK; vmwrite(GUEST_CR3, pml4_pa); - guest_cr4 = vmreadz(GUEST_CR4); + guest_cr4 = vmread(GUEST_CR4); guest_cr4 &= ~X86_CR4_LA57; vmwrite(GUEST_CR4, guest_cr4); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); - exit_reason = vmreadz(VM_EXIT_REASON); + exit_reason = vmread(VM_EXIT_REASON); GUEST_ASSERT(exit_reason == EXIT_REASON_VMCALL); } @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) } TEST_ASSERT(uc.args[1] == stage, - "Expected stage %d, got stage %lu", stage, (ulong)uc.args[1]); + "Expected stage %d, got stage %lu", stage, (unsigned long)uc.args[1]); if (stage == 1) { pr_info("L2 is active; performing save/restore.\n"); state = vcpu_save_state(vcpu);
diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index eb8021c..b9ee9d3 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
@@ -71,9 +71,9 @@ void l1_guest_code(struct vmx_pages *vmx_pages) u64 l1_tsc_deadline, l2_tsc_deadline; GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); @@ -90,19 +90,17 @@ void l1_guest_code(struct vmx_pages *vmx_pages) !(ctrl_exit_rev.clr & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) return; - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); /* * Turn on PIN control and resume the guest */ - GUEST_ASSERT(!vmwrite(PIN_BASED_VM_EXEC_CONTROL, - vmreadz(PIN_BASED_VM_EXEC_CONTROL) | - PIN_BASED_VMX_PREEMPTION_TIMER)); + vmwrite(PIN_BASED_VM_EXEC_CONTROL, + vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_VMX_PREEMPTION_TIMER); - GUEST_ASSERT(!vmwrite(VMX_PREEMPTION_TIMER_VALUE, - PREEMPTION_TIMER_VALUE)); + vmwrite(VMX_PREEMPTION_TIMER_VALUE, PREEMPTION_TIMER_VALUE); vmx_pt_rate = rdmsr(MSR_IA32_VMX_MISC) & 0x1F; @@ -110,7 +108,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) l1_vmx_pt_start = (rdtsc() >> vmx_pt_rate) << vmx_pt_rate; - GUEST_ASSERT(!vmresume()); + vmresume(); l1_vmx_pt_finish = rdtsc(); @@ -123,7 +121,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) /* * Ensure the exit from L2 is due to preemption timer expiry */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); l1_tsc_deadline = l1_vmx_pt_start + (PREEMPTION_TIMER_VALUE << vmx_pt_rate); @@ -193,7 +191,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); /* * If this stage 2 then we should verify the vmx pt expiry * is as expected. @@ -234,7 +232,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } done:
diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 469e3ab..2362bf2 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
@@ -233,13 +233,10 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, { long pages_not_moved; unsigned long nodemask = 0; - unsigned long nodemasks[sizeof(nodemask) * 8]; int nodes = 0; time_t start_time, last_update, now; time_t interval_secs = 1; - int i; int from, to; - unsigned long bit; u64 hlt_count; u64 wake_count; u64 ipis_sent; @@ -247,32 +244,19 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, fprintf(stderr, "Calling migrate_pages every %d microseconds\n", delay_usecs); - /* Get set of first 64 numa nodes available */ - kvm_get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8, - 0, MPOL_F_MEMS_ALLOWED); + nodes = kvm_get_numa_memory_nodes(&nodemask); + TEST_ASSERT(nodes > 1, + "NUMA nodes disappeared? nodemask = 0x%lx", nodemask); fprintf(stderr, "Numa nodes found amongst first %lu possible nodes " "(each 1-bit indicates node is present): %#lx\n", - sizeof(nodemask) * 8, nodemask); - - /* Init array of masks containing a single-bit in each, one for each - * available node. migrate_pages called below requires specifying nodes - * as bit masks. - */ - for (i = 0, bit = 1; i < sizeof(nodemask) * 8; i++, bit <<= 1) { - if (nodemask & bit) { - nodemasks[nodes] = nodemask & bit; - nodes++; - } - } - - TEST_ASSERT(nodes > 1, - "Did not find at least 2 numa nodes. Can't do migration"); + BITS_PER_TYPE(nodemask), nodemask); fprintf(stderr, "Migrating amongst %d nodes found\n", nodes); - from = 0; - to = 1; + from = kvm_get_next_numa_node(nodemask, -1); + to = kvm_get_next_numa_node(nodemask, from); + start_time = time(NULL); last_update = start_time; @@ -281,6 +265,9 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, wake_count = data->wake_count; while ((int)(time(NULL) - start_time) < run_secs) { + unsigned long from_mask = BIT(from); + unsigned long to_mask = BIT(to); + data->migrations_attempted++; /* @@ -291,9 +278,8 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, * KVM_CREATE_VCPU ioctl. If that assumption ever changes this * test may break or give a false positive signal. */ - pages_not_moved = migrate_pages(0, sizeof(nodemasks[from]), - &nodemasks[from], - &nodemasks[to]); + pages_not_moved = migrate_pages(0, MAXNODE_FOR_MASK(from_mask), + &from_mask, &to_mask); if (pages_not_moved < 0) fprintf(stderr, "migrate_pages failed, errno=%d\n", errno); @@ -305,9 +291,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, data->migrations_completed++; from = to; - to++; - if (to == nodes) - to = 0; + to = kvm_get_next_numa_node(nodemask, from); now = time(NULL); if (((now - start_time) % interval_secs == 0) && @@ -366,26 +350,17 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, } } -int main(int argc, char *argv[]) +static void test_xapic_ipi(int run_secs, int delay_usecs, bool migrate) { int wait_secs; const int max_halter_wait = 10; - int run_secs = 0; - int delay_usecs = 0; struct test_data_page *data; gva_t test_data_page_gva; - bool migrate = false; pthread_t threads[2]; struct thread_params params[2]; struct kvm_vm *vm; u64 *pipis_rcvd; - get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs); - if (run_secs <= 0) - run_secs = DEFAULT_RUN_SECS; - if (delay_usecs <= 0) - delay_usecs = DEFAULT_DELAY_USECS; - vm = vm_create_with_one_vcpu(¶ms[0].vcpu, halter_guest_code); vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler); @@ -473,5 +448,30 @@ int main(int argc, char *argv[]) kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + bool force_migrate = false; + unsigned long nodemask; + int run_secs = 0; + int delay_usecs = 0; + + get_cmdline_args(argc, argv, &run_secs, &force_migrate, &delay_usecs); + if (run_secs <= 0) + run_secs = DEFAULT_RUN_SECS; + if (delay_usecs <= 0) + delay_usecs = DEFAULT_DELAY_USECS; + + if (!force_migrate) + test_xapic_ipi(run_secs, delay_usecs, false); + + if (kvm_get_numa_memory_nodes(&nodemask) > 1) + test_xapic_ipi(run_secs, delay_usecs, true); + else + __TEST_REQUIRE(!force_migrate, + "Need at least 2 NUMA nodes to do migration (nodemask = 0x%lx)", + nodemask); + return 0; }
diff --git a/tools/testing/selftests/livepatch/functions.sh b/tools/testing/selftests/livepatch/functions.sh index a65b7b1..429fce1 100644 --- a/tools/testing/selftests/livepatch/functions.sh +++ b/tools/testing/selftests/livepatch/functions.sh
@@ -320,9 +320,9 @@ # find new kernel messages since the test started. local last_dmesg_msg="livepatch kselftest timestamp: $(date --rfc-3339=ns)" log "$last_dmesg_msg" - loop_until 'dmesg | grep -q "$last_dmesg_msg"' || + loop_until 'dmesg -r | grep -q "$last_dmesg_msg"' || die "buffer busy? can't find canary dmesg message: $last_dmesg_msg" - LAST_DMESG=$(dmesg | grep "$last_dmesg_msg") + LAST_DMESG=$(dmesg -r | grep "$last_dmesg_msg") echo -n "TEST: $test ... " log "===== TEST: $test =====" @@ -335,12 +335,18 @@ local result # Test results include any new dmesg entry since LAST_DMESG, then: + # - exclude debug messages except with "livepatch:" prefix # - include lines matching keywords # - exclude lines matching keywords # - filter out dmesg timestamp prefixes - result=$(dmesg | awk -v last_dmesg="$LAST_DMESG" 'p; $0 == last_dmesg { p=1 }' | \ + # - exclude the delayed kobject release messages when CONFIG_DEBUG_KOBJECT_RELEASE is on + result=$(dmesg -r | \ + awk -v last_dmesg="$LAST_DMESG" \ + 'p { if ($0 !~ /^<7>/ || $0 ~ /livepatch:/) print; next } $0 == last_dmesg { p = 1 }' | \ grep -e 'livepatch:' -e 'test_klp' | \ grep -v '\(tainting\|taints\) kernel' | \ + grep -v 'kobject: .*parent.*(delayed' | \ + sed 's/^<[0-9]*>//' | \ sed 's/^\[[ 0-9.]*\] //' | \ sed 's/^\[[ ]*[CT][0-9]*\] //')
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index df215e1..9180b9be 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile
@@ -107,7 +107,6 @@ stress_reuseport_listen.sh \ tcp_ecmp_failover.sh \ tcp_fastopen_backup_key.sh \ - test_bpf.sh \ test_bridge_backup_port.sh \ test_bridge_neigh_suppress.sh \ test_ingress_egress_chaining.sh \
diff --git a/tools/testing/selftests/net/config b/tools/testing/selftests/net/config index cb8cb46..d4416f8 100644 --- a/tools/testing/selftests/net/config +++ b/tools/testing/selftests/net/config
@@ -130,7 +130,6 @@ CONFIG_TCP_CONG_DCTCP=y CONFIG_TCP_MD5SIG=y CONFIG_TEST_BLACKHOLE_DEV=m -CONFIG_TEST_BPF=m CONFIG_TLS=m CONFIG_TRACEPOINTS=y CONFIG_TUN=y
diff --git a/tools/testing/selftests/net/psock_lib.h b/tools/testing/selftests/net/psock_lib.h index 067265b..fe85acf 100644 --- a/tools/testing/selftests/net/psock_lib.h +++ b/tools/testing/selftests/net/psock_lib.h
@@ -30,7 +30,7 @@ static __maybe_unused void pair_udp_setfilter(int fd) * ip proto udp and * skb->len == DATA_LEN and * udp[38] == 'a' or udp[38] == 'b' - * It can be generated from the following bpf_asm input: + * It corresponds to the following classic BPF assembly: * ldh [12] * jne #0x800, drop ; ETH_P_IP * ldb [23]
diff --git a/tools/testing/selftests/net/test_bpf.sh b/tools/testing/selftests/net/test_bpf.sh deleted file mode 100755 index 6567790..0000000 --- a/tools/testing/selftests/net/test_bpf.sh +++ /dev/null
@@ -1,11 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# Runs bpf test using test_bpf kernel module - -if /sbin/modprobe -q test_bpf ; then - /sbin/modprobe -q -r test_bpf; - echo "test_bpf: ok"; -else - echo "test_bpf: [FAIL]"; - exit 1; -fi
diff --git a/tools/testing/selftests/vfio/.gitignore b/tools/testing/selftests/vfio/.gitignore index 7fadc19..fc14bc1 100644 --- a/tools/testing/selftests/vfio/.gitignore +++ b/tools/testing/selftests/vfio/.gitignore
@@ -8,3 +8,4 @@ !*.mk !.gitignore !Makefile +!settings
diff --git a/tools/testing/selftests/vfio/lib/drivers/dsa/dsa.c b/tools/testing/selftests/vfio/lib/drivers/dsa/dsa.c index 19d9630..40b8541 100644 --- a/tools/testing/selftests/vfio/lib/drivers/dsa/dsa.c +++ b/tools/testing/selftests/vfio/lib/drivers/dsa/dsa.c
@@ -418,6 +418,7 @@ static void dsa_send_msi(struct vfio_pci_device *device) const struct vfio_pci_driver_ops dsa_ops = { .name = "dsa", + .region_size = sizeof(struct dsa_state), .probe = dsa_probe, .init = dsa_init, .remove = dsa_remove,
diff --git a/tools/testing/selftests/vfio/lib/drivers/igb/igb.c b/tools/testing/selftests/vfio/lib/drivers/igb/igb.c index fd9e05d..2fafcc9 100644 --- a/tools/testing/selftests/vfio/lib/drivers/igb/igb.c +++ b/tools/testing/selftests/vfio/lib/drivers/igb/igb.c
@@ -576,6 +576,7 @@ static void igb_send_msi(struct vfio_pci_device *device) const struct vfio_pci_driver_ops igb_ops = { .name = "igb", + .region_size = sizeof(struct igb), .probe = igb_probe, .init = igb_init, .remove = igb_remove,
diff --git a/tools/testing/selftests/vfio/lib/drivers/ioat/ioat.c b/tools/testing/selftests/vfio/lib/drivers/ioat/ioat.c index a871b93..c9b2836 100644 --- a/tools/testing/selftests/vfio/lib/drivers/ioat/ioat.c +++ b/tools/testing/selftests/vfio/lib/drivers/ioat/ioat.c
@@ -226,6 +226,7 @@ static void ioat_send_msi(struct vfio_pci_device *device) const struct vfio_pci_driver_ops ioat_ops = { .name = "ioat", + .region_size = sizeof(struct ioat_state), .probe = ioat_probe, .init = ioat_init, .remove = ioat_remove,
diff --git a/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5.c b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5.c new file mode 100644 index 0000000..a65b52e --- /dev/null +++ b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5.c
@@ -0,0 +1,1928 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * mlx5 VFIO selftest driver + * + * Programs mlx5 ConnectX VFs and PFs through the bare-metal command interface + * and RDMA WRITE self-loopback to perform DMA. Implements vfio_pci_driver_ops + * (probe/init/remove) and plugs into the VFIO selftest framework. + */ +#include <sched.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include <linux/errno.h> +#include <linux/io.h> +#include <linux/log2.h> +#include <linux/pci_regs.h> +#include <linux/pci_ids.h> + +#include <libvfio.h> + +#include "mlx5_hw.h" + +/* + * Driver state — overlaid on device->driver.region.vaddr. + * + * Contains both software-only state and HW-visible DMA buffers. HW buffers need + * strict IOVA alignment. + */ +struct mlx5st_device { + /* Back pointer */ + struct vfio_pci_device *device; + + /* BAR0 */ + struct mlx5st_initial_seg __iomem *init_seg; + void __iomem *bar0; + + /* Command interface */ + struct mlx5st_cmd_queue_entry *cmd_lay; + struct mlx5st_cmd_queue_entry *pages_cmd_lay; + u8 cmd_log_stride; + unsigned int pages_slot; + u8 cmd_token; + bool cmd_sig_enabled; + + /* PD */ + u32 pdn; + + /* Global PA-mode MKEY */ + u32 global_lkey; + u32 global_rkey; + u32 mkey_index; + + /* CQ */ + u32 cqn; + u32 cq_ci; + u32 cq_arm_sn; + + /* UAR */ + u32 uar_page; + void __iomem *uar_base; + unsigned int uar_bf_offset; + + /* EQ (cmd/pages events — polled, not interrupt-driven) */ + u32 eqn; + u32 eq_cons_index; + bool have_eq; + + /* MSI EQ (CQ completion events — fires MSI-X) */ + u32 msi_eqn; + u32 msi_eq_cons_index; + bool have_msi_eq; + + /* Async pages slot state */ + bool pages_slot_in_use; + bool pages_slot_is_reclaim; + unsigned int pages_reclaim_npages; + unsigned int pages_pending_give; + unsigned int pages_pending_reclaim; + u16 pages_pending_func_id; + bool pages_func_id_seen; + + /* QP */ + u32 qpn; + u32 sq_pi; + u32 sq_ci; + + /* FW pages bitmap */ + u64 fw_pages_bitmap[MAX_FW_PAGES / 64]; + u32 fw_pages_given; + u16 fw_func_id; + + /* Capabilities */ + bool fl_supported; + u8 log_max_msg; + + /* Buffers used by send_msi() to trigger an interrupt */ + u64 send_msi_src; + u64 send_msi_dst; + + /* + * HW-visible DMA buffers below — device reads/writes via DMA. + */ + struct mlx5st_cmd_queue_entry cmd_queue + [MLX5_HW_PAGE_SIZE / sizeof(struct mlx5st_cmd_queue_entry)] + __aligned(MLX5_HW_PAGE_SIZE); + struct mlx5st_send_wqe sq_buf[SQ_WQE_CNT]; + struct mlx5st_dbrec cq_dbrec; + struct mlx5st_dbrec qp_dbrec; + struct mlx5st_cqe64 cq_buf[CQ_CQE_CNT]; + + /* Slot 0 mailboxes (regular commands) */ + struct mlx5st_mbox_entry cmd_in_mbox[CMD_MBOX_NENT]; + struct mlx5st_mbox_entry cmd_out_mbox[CMD_MBOX_NENT]; + + /* Pages slot mailboxes (async MANAGE_PAGES) */ + struct mlx5st_mbox_entry pages_in_mbox[CMD_MBOX_NENT]; + struct mlx5st_mbox_entry pages_out_mbox[CMD_MBOX_NENT]; + + /* EQ does not support page_offset */ + struct mlx5st_eqe eq_buf[EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE); + + /* MSI EQ buffer — CQ completions generate EQEs here -> MSI-X */ + struct mlx5st_eqe msi_eq_buf[MSI_EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE); + + u8 fw_pages[MAX_FW_PAGES][MLX5_HW_PAGE_SIZE] + __aligned(MLX5_HW_PAGE_SIZE); +}; + +/* Check against HW limits on IOVA alignment */ +static_assert(offsetof(struct mlx5st_device, cmd_in_mbox) % + CMD_MBOX_STRIDE == 0, + "cmd_in_mbox must be stride-aligned"); +static_assert(offsetof(struct mlx5st_device, pages_in_mbox) % + CMD_MBOX_STRIDE == 0, + "pages_in_mbox must be stride-aligned"); +static_assert(offsetof(struct mlx5st_device, cq_buf) % 64 == 0, + "cq_buf must be 64-byte aligned"); +static_assert(offsetof(struct mlx5st_device, sq_buf) % 64 == 0, + "sq_buf must be 64-byte aligned"); +static_assert(offsetof(struct mlx5st_device, cq_dbrec) % 64 == 0, + "cq_dbrec must be 64-byte aligned"); +static_assert(offsetof(struct mlx5st_device, qp_dbrec) % 64 == 0, + "qp_dbrec must be 64-byte aligned"); +static_assert(offsetof(struct mlx5st_device, eq_buf) % + MLX5_HW_PAGE_SIZE == 0, + "eq_buf must be page-aligned"); +static_assert(offsetof(struct mlx5st_device, msi_eq_buf) % + MLX5_HW_PAGE_SIZE == 0, + "msi_eq_buf must be page-aligned"); +static_assert(offsetof(struct mlx5st_device, fw_pages) % + MLX5_HW_PAGE_SIZE == 0, + "fw_pages must be page-aligned"); + +static struct mlx5st_device *to_mlx5st(struct vfio_pci_device *device) +{ + return device->driver.region.vaddr; +} + +/* + * Fill a PAS (Physical Address Segment) for a buffer in the driver region. + * Sets pas[0] to the page-aligned IOVA and returns the page_offset (the + * buffer's byte offset within that page, in units of 64 bytes). + */ +static unsigned int mlx5st_fill_pas(struct vfio_pci_device *device, void *buf, + __be64 *pas) +{ + u64 iova = to_iova(device, buf); + + pas[0] = cpu_to_be64(iova & ~(u64)(MLX5_HW_PAGE_SIZE - 1)); + return (iova & (MLX5_HW_PAGE_SIZE - 1)) / 64; +} + +/* + * Probe — match mlx5 devices by PCI vendor/device ID. + */ + +static int mlx5st_probe(struct vfio_pci_device *device) +{ + static const u16 mlx5st_pci_ids[] = { + 0x1011, /* Connect-IB */ + 0x1012, /* Connect-IB VF */ + 0x1013, /* ConnectX-4 */ + 0x1014, /* ConnectX-4 VF */ + 0x1015, /* ConnectX-4LX */ + 0x1016, /* ConnectX-4LX VF */ + 0x1017, /* ConnectX-5 */ + 0x1018, /* ConnectX-5 VF */ + 0x1019, /* ConnectX-5 Ex */ + 0x101a, /* ConnectX-5 Ex VF */ + 0x101b, /* ConnectX-6 */ + 0x101c, /* ConnectX-6 VF */ + 0x101d, /* ConnectX-6 Dx */ + 0x101e, /* ConnectX-6 Dx VF */ + 0x101f, /* ConnectX-6 LX */ + 0x1021, /* ConnectX-7 */ + 0x1023, /* ConnectX-8 */ + 0x1025, /* ConnectX-9 */ + 0x1027, /* ConnectX-10 */ + 0x2101, /* ConnectX-10 NVLink-C2C */ + 0xa2d2, /* BlueField integrated ConnectX-5 */ + 0xa2d3, /* BlueField integrated ConnectX-5 VF */ + 0xa2d6, /* BlueField-2 integrated ConnectX-6 Dx */ + 0xa2dc, /* BlueField-3 integrated ConnectX-7 */ + 0xa2df, /* BlueField-4 integrated ConnectX-8 */ + }; + unsigned int i; + u16 did; + + if (vfio_pci_config_readw(device, PCI_VENDOR_ID) != + PCI_VENDOR_ID_MELLANOX) + return -ENODEV; + + did = vfio_pci_config_readw(device, PCI_DEVICE_ID); + for (i = 0; i < ARRAY_SIZE(mlx5st_pci_ids); i++) { + if (mlx5st_pci_ids[i] == did) + return 0; + } + + return -ENODEV; +} + +/* + * Command interface + */ + +static u8 xor8_buf(const void *buf, size_t offset, size_t len) +{ + const u8 *p = buf; + u8 sum = 0; + size_t i; + + for (i = offset; i < offset + len; i++) + sum ^= p[i]; + return sum; +} + +#define CMD_IF_BOX_CTRL_OFF MLX5_BYTE_OFF(cmd_if_box, reserved_at_1000) +#define CMD_IF_BOX_CTRL_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, ctrl_signature) +#define CMD_IF_BOX_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, signature) + +static void mlx5st_cmd_calc_block_sig(struct mlx5st_cmd_if_box *blk) +{ + MLX5_SET(cmd_if_box, blk, ctrl_signature, + ~xor8_buf(blk, CMD_IF_BOX_CTRL_OFF, + CMD_IF_BOX_CTRL_SIG_OFF - CMD_IF_BOX_CTRL_OFF)); + MLX5_SET(cmd_if_box, blk, signature, + ~xor8_buf(blk, 0, CMD_IF_BOX_SIG_OFF)); +} + +static int mlx5st_cmd_verify_block_sig(struct mlx5st_cmd_if_box *blk) +{ + if (xor8_buf(blk, CMD_IF_BOX_CTRL_OFF, + CMD_IF_BOX_SIG_OFF - CMD_IF_BOX_CTRL_OFF) != 0xff) + return -1; + if (xor8_buf(blk, 0, sizeof(struct mlx5st_cmd_if_box)) != 0xff) + return -1; + return 0; +} + +static unsigned int mlx5st_cmd_setup_mbox_chain(struct vfio_pci_device *device, + struct mlx5st_mbox_entry *mbox, + unsigned int nblocks, u8 token) +{ + unsigned int i; + + for (i = 0; i < nblocks; i++) { + struct mlx5st_cmd_if_box *blk = &mbox[i].block; + u64 next_iova; + + memset(blk, 0, sizeof(struct mlx5st_cmd_if_box)); + MLX5_SET(cmd_if_box, blk, block_number, i); + MLX5_SET(cmd_if_box, blk, token, token); + if (i < nblocks - 1) { + next_iova = to_iova(device, &mbox[i + 1]); + MLX5_SET(cmd_if_box, blk, next_pointer_63_32, + next_iova >> 32); + MLX5_SET(cmd_if_box, blk, next_pointer_31_10, + (u32)next_iova >> 10); + } + } + return nblocks; +} + +static void mlx5st_cmd_copy_to_mbox(struct mlx5st_mbox_entry *mbox, + const void *data, unsigned int len) +{ + const u8 *src = data; + unsigned int i = 0; + + while (len > 0) { + unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ? + len : + MLX5_CMD_DATA_BLOCK_SIZE; + + memcpy(MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data), + src, chunk); + src += chunk; + len -= chunk; + i++; + } +} + +static void mlx5st_cmd_copy_from_mbox(void *data, + const struct mlx5st_mbox_entry *mbox, + unsigned int len) +{ + unsigned int i = 0; + u8 *dst = data; + + while (len > 0) { + unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ? + len : + MLX5_CMD_DATA_BLOCK_SIZE; + + memcpy(dst, + MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data), + chunk); + dst += chunk; + len -= chunk; + i++; + } +} + +/* Forward declaration — cmd_exec polls events during command wait */ +static void mlx5st_process_events(struct mlx5st_device *dev); + +static const char *mlx5st_cmd_name(u16 opcode) +{ + switch (opcode) { + case MLX5_CMD_OP_QUERY_HCA_CAP: return "QUERY_HCA_CAP"; + case MLX5_CMD_OP_INIT_HCA: return "INIT_HCA"; + case MLX5_CMD_OP_TEARDOWN_HCA: return "TEARDOWN_HCA"; + case MLX5_CMD_OP_ENABLE_HCA: return "ENABLE_HCA"; + case MLX5_CMD_OP_DISABLE_HCA: return "DISABLE_HCA"; + case MLX5_CMD_OP_QUERY_PAGES: return "QUERY_PAGES"; + case MLX5_CMD_OP_MANAGE_PAGES: return "MANAGE_PAGES"; + case MLX5_CMD_OP_SET_HCA_CAP: return "SET_HCA_CAP"; + case MLX5_CMD_OP_SET_ISSI: return "SET_ISSI"; + case MLX5_CMD_OP_CREATE_MKEY: return "CREATE_MKEY"; + case MLX5_CMD_OP_DESTROY_MKEY: return "DESTROY_MKEY"; + case MLX5_CMD_OP_CREATE_EQ: return "CREATE_EQ"; + case MLX5_CMD_OP_DESTROY_EQ: return "DESTROY_EQ"; + case MLX5_CMD_OP_CREATE_CQ: return "CREATE_CQ"; + case MLX5_CMD_OP_DESTROY_CQ: return "DESTROY_CQ"; + case MLX5_CMD_OP_CREATE_QP: return "CREATE_QP"; + case MLX5_CMD_OP_DESTROY_QP: return "DESTROY_QP"; + case MLX5_CMD_OP_RST2INIT_QP: return "RST2INIT_QP"; + case MLX5_CMD_OP_INIT2RTR_QP: return "INIT2RTR_QP"; + case MLX5_CMD_OP_RTR2RTS_QP: return "RTR2RTS_QP"; + case MLX5_CMD_OP_ALLOC_PD: return "ALLOC_PD"; + case MLX5_CMD_OP_DEALLOC_PD: return "DEALLOC_PD"; + case MLX5_CMD_OP_ALLOC_UAR: return "ALLOC_UAR"; + case MLX5_CMD_OP_DEALLOC_UAR: return "DEALLOC_UAR"; + default: return "UNKNOWN"; + } +} + +/* + * Post a command on a given slot: fill the cmd_queue_entry, set up mailbox + * chains, compute signatures, hand ownership to FW, and ring the doorbell. + */ +static void mlx5st_cmd_post(struct mlx5st_device *dev, + struct mlx5st_cmd_queue_entry *cmd, + struct mlx5st_mbox_entry *in_mbox, + struct mlx5st_mbox_entry *out_mbox, + void *in, unsigned int ilen, unsigned int olen, + u32 doorbell) +{ + struct vfio_pci_device *device = dev->device; + unsigned int in_remain, out_remain, in_nblk, out_nblk; + unsigned int i; + void *cin, *cout; + u8 token; + + /* Rotating non-zero token ties cmd entry to its mailbox blocks */ + token = ++dev->cmd_token; + if (!token) + token = ++dev->cmd_token; + + in_remain = ilen > MLX5_CMD_INLINE_SZ ? ilen - MLX5_CMD_INLINE_SZ : 0; + out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0; + in_nblk = (in_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / + MLX5_CMD_DATA_BLOCK_SIZE; + out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / + MLX5_CMD_DATA_BLOCK_SIZE; + + /* Set up mailbox chains */ + if (in_nblk > 0) { + mlx5st_cmd_setup_mbox_chain(device, in_mbox, in_nblk, token); + mlx5st_cmd_copy_to_mbox(in_mbox, + (u8 *)in + MLX5_CMD_INLINE_SZ, + in_remain); + } + if (out_nblk > 0) + mlx5st_cmd_setup_mbox_chain(device, out_mbox, out_nblk, token); + + /* Copy inline input */ + cin = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_input_inline_data); + memset(cin, 0, MLX5_CMD_INLINE_SZ); + memcpy(cin, in, ilen < MLX5_CMD_INLINE_SZ ? ilen : MLX5_CMD_INLINE_SZ); + MLX5_SET(cmd_queue_entry, cmd, input_length, ilen); + MLX5_SET(cmd_queue_entry, cmd, token, token); + + /* Zero inline output */ + cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); + memset(cout, 0, MLX5_CMD_INLINE_SZ); + MLX5_SET(cmd_queue_entry, cmd, output_length, olen); + + /* + * Compute signatures: mailbox blocks first, then cmd_queue_entry. + * The sig must cover the final state including ownership=0x1, but + * we must not set ownership until after the sig is in place - + * XOR in the 0x1 without storing it to memory. + */ + for (i = 0; i < in_nblk; i++) + mlx5st_cmd_calc_block_sig(&in_mbox[i].block); + for (i = 0; i < out_nblk; i++) + mlx5st_cmd_calc_block_sig(&out_mbox[i].block); + MLX5_SET(cmd_queue_entry, cmd, signature, 0); + MLX5_SET(cmd_queue_entry, cmd, signature, + ~(xor8_buf(cmd, 0, sizeof(struct mlx5st_cmd_queue_entry)) ^ + 0x1)); + + /* Ensure all cmd data (including sig) is visible, then hand to FW */ + dma_wmb(); + MLX5_SET_ONCE(cmd_queue_entry, cmd, ownership, 1); + + /* Ring doorbell */ + MLX5_SET_MMIO(initial_seg, dev->init_seg, command_doorbell_vector, + doorbell); +} + +static void mlx5st_cmd_exec(struct mlx5st_device *dev, void *in, + unsigned int ilen, void *out, unsigned int olen) +{ + struct mlx5st_cmd_queue_entry *cmd = dev->cmd_lay; + unsigned int out_remain, out_nblk; + struct timespec start, now; + unsigned int elapsed; + unsigned int i; + void *cout; + + mlx5st_cmd_post(dev, cmd, dev->cmd_in_mbox, dev->cmd_out_mbox, in, + ilen, olen, 1); + + out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0; + out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / + MLX5_CMD_DATA_BLOCK_SIZE; + + /* Poll for completion — also process EQ events for PF page requests */ + clock_gettime(CLOCK_MONOTONIC, &start); + for (;;) { + if (!MLX5_GET_ONCE(cmd_queue_entry, cmd, ownership)) + break; + if (dev->have_eq) + mlx5st_process_events(dev); + sched_yield(); + clock_gettime(CLOCK_MONOTONIC, &now); + elapsed = (now.tv_sec - start.tv_sec) * 1000 + + (now.tv_nsec - start.tv_nsec) / 1000000; + if (elapsed > MLX5_CMD_TIMEOUT_MS) + VFIO_FAIL("cmd timeout after %d ms", elapsed); + } + /* Ensure output data reads happen after ownership is seen clear */ + dma_rmb(); + + /* Verify output signatures when FW has checksums enabled */ + if (dev->cmd_sig_enabled) { + if (xor8_buf(cmd, 0, + sizeof(struct mlx5st_cmd_queue_entry)) != 0xff) + VFIO_FAIL("cmd output signature mismatch"); + for (i = 0; i < out_nblk; i++) { + if (mlx5st_cmd_verify_block_sig( + &dev->cmd_out_mbox[i].block)) + VFIO_FAIL("cmd output mailbox block %d signature mismatch", + i); + } + } + + /* Copy output: inline first */ + cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); + memcpy(out, cout, olen < MLX5_CMD_INLINE_SZ ? olen : MLX5_CMD_INLINE_SZ); + + /* Copy remaining from output mailbox chain */ + if (out_remain > 0) + mlx5st_cmd_copy_from_mbox((u8 *)out + MLX5_CMD_INLINE_SZ, + dev->cmd_out_mbox, out_remain); + + /* Check command status */ + if (MLX5_GET(enable_hca_out, out, status) != MLX5_CMD_STAT_OK) + VFIO_FAIL("%s: status=0x%x syndrome=0x%x", + mlx5st_cmd_name(MLX5_GET(enable_hca_in, in, opcode)), + MLX5_GET(enable_hca_out, out, status), + MLX5_GET(enable_hca_out, out, syndrome)); +} + +static struct mlx5st_cmd_queue_entry * +mlx5st_cmd_slot_init(struct mlx5st_device *dev, unsigned int slot, + struct mlx5st_mbox_entry *in_mbox, + struct mlx5st_mbox_entry *out_mbox) +{ + struct vfio_pci_device *device = dev->device; + struct mlx5st_cmd_queue_entry *cmd = + &dev->cmd_queue[(slot << dev->cmd_log_stride) / + sizeof(struct mlx5st_cmd_queue_entry)]; + u64 iova; + + MLX5_SET(cmd_queue_entry, cmd, type, + MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT); + iova = to_iova(device, in_mbox); + MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_63_32, + iova >> 32); + MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_31_9, iova >> 9); + iova = to_iova(device, out_mbox); + MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_63_32, + iova >> 32); + MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_31_9, + iova >> 9); + return cmd; +} + +static void mlx5st_cmd_init(struct mlx5st_device *dev) +{ + struct mlx5st_initial_seg __iomem *seg = dev->init_seg; + struct vfio_pci_device *device = dev->device; + u16 cmdif_rev; + u8 log_sz; + u64 iova; + + cmdif_rev = MLX5_GET_MMIO(initial_seg, seg, cmd_interface_rev); + VFIO_ASSERT_EQ(cmdif_rev, 5); + + /* Read command queue geometry from BAR */ + log_sz = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_size); + dev->cmd_log_stride = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_stride); + dev->pages_slot = (1 << log_sz) - 1; + + VFIO_ASSERT_LE((unsigned int)(1 << log_sz), 32u); + VFIO_ASSERT_GE((unsigned int)(1 << dev->cmd_log_stride), + (unsigned int)sizeof(struct mlx5st_cmd_queue_entry)); + VFIO_ASSERT_LE((unsigned int)((dev->pages_slot + 1) << + dev->cmd_log_stride), + (unsigned int)sizeof(dev->cmd_queue)); + + /* Set up slot 0 — regular commands */ + dev->cmd_lay = mlx5st_cmd_slot_init(dev, 0, dev->cmd_in_mbox, + dev->cmd_out_mbox); + + /* Set up pages slot — async MANAGE_PAGES */ + dev->pages_cmd_lay = mlx5st_cmd_slot_init(dev, dev->pages_slot, + dev->pages_in_mbox, + dev->pages_out_mbox); + + /* Write command queue page address to BAR0 */ + iova = to_iova(device, dev->cmd_queue); + MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_63_32, iova >> 32); + MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_31_12, iova >> 12); + + dev_dbg(device, + "Command interface initialized (cmdif_rev=5, log_sz=%u, log_stride=%u, pages_slot=%u)\n", + log_sz, dev->cmd_log_stride, dev->pages_slot); +} + +/* + * FW pages: bitmap allocator + MANAGE_PAGES + */ + +static void mlx5st_fw_pages_alloc(struct mlx5st_device *dev, + unsigned int npages, u64 *iovas) +{ + struct vfio_pci_device *device = dev->device; + unsigned int found = 0; + unsigned int w, b; + u64 word; + + for (w = 0; w < MAX_FW_PAGES / 64 && found < npages; w++) { + word = dev->fw_pages_bitmap[w]; + + for (b = 0; b < 64 && found < npages; b++) { + if (!(word & (1ULL << b))) { + unsigned int idx = w * 64 + b; + + dev->fw_pages_bitmap[w] |= (1ULL << b); + iovas[found++] = to_iova(device, + dev->fw_pages[idx]); + } + } + } + VFIO_ASSERT_EQ(found, npages); + dev->fw_pages_given += npages; +} + +static void mlx5st_fw_pages_free(struct mlx5st_device *dev, + unsigned int npages, const u64 *iovas) +{ + struct vfio_pci_device *device = dev->device; + unsigned int i, idx; + u64 off; + + for (i = 0; i < npages; i++) { + off = iovas[i] - to_iova(device, dev->fw_pages); + idx = off / MLX5_HW_PAGE_SIZE; + + VFIO_ASSERT_TRUE(idx < MAX_FW_PAGES); + dev->fw_pages_bitmap[idx / 64] &= ~(1ULL << (idx % 64)); + } + dev->fw_pages_given -= npages; +} + +static void *mlx5st_build_manage_pages_give(u16 func_id, unsigned int npages, + const u64 *iovas, + unsigned int *out_inlen) +{ + unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in) + npages * 8; + unsigned int i; + void *in; + + in = calloc(1, inlen); + VFIO_ASSERT_NOT_NULL(in); + + MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); + MLX5_SET(manage_pages_in, in, op_mod, + MLX5_MANAGE_PAGES_IN_OP_MOD_ALLOCATION_SUCCESS); + MLX5_SET(manage_pages_in, in, function_id, func_id); + MLX5_SET(manage_pages_in, in, input_num_entries, npages); + + for (i = 0; i < npages; i++) + MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, iovas[i]); + + *out_inlen = inlen; + return in; +} + +static void mlx5st_fw_pages_give_one(struct mlx5st_device *dev, u16 func_id, + unsigned int npages, u64 *iovas) +{ + u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {}; + unsigned int inlen; + void *in; + + in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen); + mlx5st_cmd_exec(dev, in, inlen, out, sizeof(out)); + free(in); +} + +static void mlx5st_fw_pages_give(struct mlx5st_device *dev, u16 func_id, + unsigned int npages) +{ + unsigned int remaining = npages; + u64 *iovas; + + if (!npages) + return; + + iovas = calloc(npages, sizeof(u64)); + VFIO_ASSERT_NOT_NULL(iovas); + + mlx5st_fw_pages_alloc(dev, npages, iovas); + + /* Batch into chunks that fit in one mailbox */ + for (unsigned int off = 0; remaining > 0;) { + unsigned int batch = remaining < MAX_FW_PAGES_PER_CMD ? + remaining : + MAX_FW_PAGES_PER_CMD; + + mlx5st_fw_pages_give_one(dev, func_id, batch, iovas + off); + off += batch; + remaining -= batch; + } + + dev_dbg(dev->device, "MANAGE_PAGES GIVE: %d pages to func_id=%u\n", + npages, func_id); + free(iovas); +} + +static void mlx5st_fw_pages_satisfy(struct mlx5st_device *dev, int boot) +{ + u32 qo[MLX5_ST_SZ_DW(query_pages_out)] = {}; + u32 qi[MLX5_ST_SZ_DW(query_pages_in)] = {}; + u16 func_id; + int npages; + + MLX5_SET(query_pages_in, qi, opcode, MLX5_CMD_OP_QUERY_PAGES); + MLX5_SET(query_pages_in, qi, op_mod, boot ? 0x01 : 0x02); + mlx5st_cmd_exec(dev, qi, sizeof(qi), qo, sizeof(qo)); + + npages = MLX5_GET(query_pages_out, qo, num_pages); + func_id = MLX5_GET(query_pages_out, qo, function_id); + dev_dbg(dev->device, "QUERY_PAGES (%s): %d pages (func_id=%u)\n", + boot ? "boot" : "init", npages, func_id); + + if (npages > 0) { + dev->fw_func_id = func_id; + mlx5st_fw_pages_give(dev, func_id, npages); + } +} + +/* + * Async MANAGE_PAGES on the pages command slot. + * + * On PFs, firmware sends PAGE_REQUEST events via the EQ during command + * execution. We must respond with MANAGE_PAGES on a second command slot + * before the first (regular) command can complete. + */ + +static void mlx5st_pages_slot_post(struct mlx5st_device *dev, void *in, + unsigned int ilen, unsigned int olen) +{ + mlx5st_cmd_post(dev, dev->pages_cmd_lay, dev->pages_in_mbox, + dev->pages_out_mbox, in, ilen, olen, + 1 << dev->pages_slot); +} + +static void mlx5st_pages_slot_give(struct mlx5st_device *dev, u16 func_id, + unsigned int npages) +{ + unsigned int inlen; + u64 *iovas; + void *in; + + iovas = calloc(npages, sizeof(u64)); + VFIO_ASSERT_NOT_NULL(iovas); + + mlx5st_fw_pages_alloc(dev, npages, iovas); + + in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen); + free(iovas); + + mlx5st_pages_slot_post(dev, in, inlen, + MLX5_ST_SZ_BYTES(manage_pages_out)); + dev->pages_slot_in_use = true; + dev->pages_slot_is_reclaim = false; + free(in); + + dev_dbg(dev->device, + "PAGE_REQUEST: %d pages given async to func_id=%u\n", + npages, func_id); +} + +static void mlx5st_pages_slot_reclaim(struct mlx5st_device *dev, u16 func_id, + unsigned int npages) +{ + unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); + unsigned int outlen = + MLX5_ST_SZ_BYTES(manage_pages_out) + npages * 8; + void *in; + + in = calloc(1, inlen); + VFIO_ASSERT_NOT_NULL(in); + + MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); + MLX5_SET(manage_pages_in, in, op_mod, + MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES); + MLX5_SET(manage_pages_in, in, function_id, func_id); + MLX5_SET(manage_pages_in, in, input_num_entries, npages); + + mlx5st_pages_slot_post(dev, in, inlen, outlen); + dev->pages_slot_in_use = true; + dev->pages_slot_is_reclaim = true; + dev->pages_reclaim_npages = npages; + free(in); + + dev_dbg(dev->device, + "PAGE_REQUEST: reclaim %d pages async from func_id=%u\n", + npages, func_id); +} + +static void mlx5st_pages_slot_kick(struct mlx5st_device *dev) +{ + unsigned int batch; + + if (dev->pages_slot_in_use) + return; + + if (dev->pages_pending_give) { + batch = dev->pages_pending_give < MAX_FW_PAGES_PER_CMD ? + dev->pages_pending_give : + MAX_FW_PAGES_PER_CMD; + dev->pages_pending_give -= batch; + mlx5st_pages_slot_give(dev, dev->pages_pending_func_id, batch); + } else if (dev->pages_pending_reclaim) { + batch = dev->pages_pending_reclaim < MAX_FW_PAGES_PER_CMD ? + dev->pages_pending_reclaim : + MAX_FW_PAGES_PER_CMD; + dev->pages_pending_reclaim -= batch; + mlx5st_pages_slot_reclaim(dev, dev->pages_pending_func_id, + batch); + } +} + +static void mlx5st_fw_pages_give_async(struct mlx5st_device *dev, + u16 func_id, unsigned int npages) +{ + if (!npages) + return; + + dev->pages_pending_give += npages; + dev->pages_pending_func_id = func_id; + mlx5st_pages_slot_kick(dev); +} + +static void mlx5st_fw_pages_reclaim_async(struct mlx5st_device *dev, + u16 func_id, unsigned int npages) +{ + dev->pages_pending_reclaim += npages; + dev->pages_pending_func_id = func_id; + mlx5st_pages_slot_kick(dev); +} + +static void mlx5st_pages_slot_complete(struct mlx5st_device *dev) +{ + struct mlx5st_cmd_queue_entry *cmd = dev->pages_cmd_lay; + void *cout; + + dma_rmb(); + + cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); + if (MLX5_GET(enable_hca_out, cout, status) != MLX5_CMD_STAT_OK) + VFIO_FAIL("async MANAGE_PAGES failed: status=0x%x syndrome=0x%x", + MLX5_GET(enable_hca_out, cout, status), + MLX5_GET(enable_hca_out, cout, syndrome)); + + if (dev->pages_slot_is_reclaim) { + unsigned int outlen = MLX5_ST_SZ_BYTES(manage_pages_out) + + dev->pages_reclaim_npages * 8; + unsigned int num_claimed; + unsigned int i; + void *out; + u64 *iovas; + + out = calloc(1, outlen); + iovas = calloc(dev->pages_reclaim_npages, sizeof(u64)); + VFIO_ASSERT_NOT_NULL(out); + VFIO_ASSERT_NOT_NULL(iovas); + + /* Copy inline output */ + memcpy(out, cout, MLX5_CMD_INLINE_SZ); + if (outlen > MLX5_CMD_INLINE_SZ) + mlx5st_cmd_copy_from_mbox( + (u8 *)out + MLX5_CMD_INLINE_SZ, + dev->pages_out_mbox, + outlen - MLX5_CMD_INLINE_SZ); + + num_claimed = + MLX5_GET(manage_pages_out, out, output_num_entries); + for (i = 0; i < num_claimed; i++) + iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas, + i); + + mlx5st_fw_pages_free(dev, num_claimed, iovas); + dev_dbg(dev->device, "PAGE_REQUEST: reclaimed %d pages\n", + num_claimed); + + free(iovas); + free(out); + } + + dev->pages_slot_in_use = false; + mlx5st_pages_slot_kick(dev); +} + +/* + * UAR alloc/dealloc + */ + +static void mlx5st_alloc_uar(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {}; + u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {}; + + MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->uar_page = MLX5_GET(alloc_uar_out, out, uar); + dev->uar_base = (u8 __iomem*)dev->bar0 + dev->uar_page * MLX5_HW_PAGE_SIZE; + dev->uar_bf_offset = MLX5_BF_OFFSET; + + dev_dbg(dev->device, + "Allocated UAR page_id=%u, doorbell offset=0x%x\n", + dev->uar_page, + dev->uar_page * MLX5_HW_PAGE_SIZE + MLX5_BF_OFFSET); +} + +static void mlx5st_dealloc_uar(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)] = {}; + u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {}; + + MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR); + MLX5_SET(dealloc_uar_in, in, uar, dev->uar_page); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +/* + * EQ infrastructure + */ + +static struct mlx5st_eqe *mlx5st_eq_get_eqe(struct mlx5st_device *dev, u32 cc) +{ + u32 ci = dev->eq_cons_index + cc; + struct mlx5st_eqe *eqe = &dev->eq_buf[ci % EQ_NENT]; + u8 owner = MLX5_GET_ONCE(eqe, eqe, owner); + u8 expected = !!(ci & EQ_NENT); + + if (owner != expected) + return NULL; + dma_rmb(); + return eqe; +} + +static void mlx5st_eq_update_ci(struct mlx5st_device *dev, u32 cc, bool arm) +{ + u32 val; + + dev->eq_cons_index += cc; + val = (dev->eq_cons_index & 0xffffff) | (dev->eqn << 24); + iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET + + (arm ? 0 : 8)); +} + +static void mlx5st_create_eq(struct mlx5st_device *dev) +{ + struct vfio_pci_device *device = dev->device; + u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {}; + u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {}; + struct mlx5_ifc_eqc_bits *eqc; + unsigned int i; + __be64 *pas; + + /* Initialize EQE owner bits */ + for (i = 0; i < EQ_NENT; i++) { + struct mlx5st_eqe *eqe = &dev->eq_buf[i]; + + MLX5_SET_ONCE(eqe, eqe, owner, 1); + } + + MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); + + /* Subscribe to CMD completions and PAGE_REQUEST events */ + MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, 0, + (1ULL << MLX5_EVENT_TYPE_CMD) | + (1ULL << MLX5_EVENT_TYPE_PAGE_REQUEST)); + + eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); + MLX5_SET(eqc, eqc, log_eq_size, LOG_EQ_SIZE); + MLX5_SET(eqc, eqc, uar_page, dev->uar_page); + pas = MLX5_ADDR_OF(create_eq_in, in, pas); + VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->eq_buf, pas), 0u); + MLX5_SET(eqc, eqc, log_page_size, 0); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->eqn = MLX5_GET(create_eq_out, out, eq_number); + dev->eq_cons_index = 0; + mlx5st_eq_update_ci(dev, 0, 0); + dev->have_eq = true; + + dev_dbg(device, "Created EQ: eqn=%u, %d entries (CMD+PAGE_REQUEST)\n", + dev->eqn, EQ_NENT); +} + +static void mlx5st_destroy_eq(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {}; + u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; + + MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); + MLX5_SET(destroy_eq_in, in, eq_number, dev->eqn); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +/* + * Drain all pending EQ events. Dispatches PAGE_REQUEST to the async pages + * slot and CMD completions to the pages slot completion handler. + */ +static void mlx5st_process_events(struct mlx5st_device *dev) +{ + struct mlx5st_eqe *eqe; + u32 cc = 0; + + while ((eqe = mlx5st_eq_get_eqe(dev, cc))) { + u8 type = MLX5_GET(eqe, eqe, event_type); + + switch (type) { + case MLX5_EVENT_TYPE_PAGE_REQUEST: { + void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data); + u16 func_id = MLX5_GET(pages_req_event, evdata, + function_id); + s32 npages = (s32)MLX5_GET(pages_req_event, evdata, + num_pages); + + /* + * The selftest doesn't use more than one func_id so a + * simple counter approach is possible. + */ + if (dev->pages_func_id_seen) + VFIO_ASSERT_EQ(func_id, + dev->pages_pending_func_id); + dev->pages_func_id_seen = true; + + if (npages > 0) + mlx5st_fw_pages_give_async(dev, func_id, + npages); + else if (npages < 0) + mlx5st_fw_pages_reclaim_async(dev, func_id, + -npages); + break; + } + case MLX5_EVENT_TYPE_CMD: { + void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data); + u32 vector = MLX5_GET(cmd_inter_comp_event, evdata, + command_completion_vector); + + if (vector & (1U << dev->pages_slot)) + mlx5st_pages_slot_complete(dev); + break; + } + default: + break; + } + cc++; + } + + if (cc) + mlx5st_eq_update_ci(dev, cc, 0); +} + +/* + * MSI EQ — dedicated EQ for CQ completion events that fires MSI-X. + * Separate from the cmd/pages EQ so that only CQ completions (from + * send_msi or memcpy) trigger the interrupt vector. + */ + +static void mlx5st_msi_eq_drain(struct mlx5st_device *dev) +{ + u32 cc = 0; + u32 val; + + while (cc < MSI_EQ_NENT) { + u32 ci = dev->msi_eq_cons_index + cc; + struct mlx5st_eqe *eqe = + &dev->msi_eq_buf[ci % MSI_EQ_NENT]; + + if (MLX5_GET_ONCE(eqe, eqe, owner) != !!(ci & MSI_EQ_NENT)) + break; + cc++; + } + + /* Update consumer index and re-arm for next interrupt */ + dev->msi_eq_cons_index += cc; + val = (dev->msi_eq_cons_index & 0xffffff) | (dev->msi_eqn << 24); + iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET); +} + +static void mlx5st_create_msi_eq(struct mlx5st_device *dev) +{ + struct vfio_pci_device *device = dev->device; + u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {}; + u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {}; + struct mlx5_ifc_eqc_bits *eqc; + unsigned int i; + __be64 *pas; + + /* Initialize EQE owner bits */ + for (i = 0; i < MSI_EQ_NENT; i++) { + struct mlx5st_eqe *eqe = &dev->msi_eq_buf[i]; + + MLX5_SET_ONCE(eqe, eqe, owner, 1); + } + + MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); + + /* + * No event_bitmask — completion events are routed to this EQ via + * the CQ's c_eqn field, not through CREATE_EQ subscription. + */ + eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); + MLX5_SET(eqc, eqc, log_eq_size, LOG_MSI_EQ_SIZE); + MLX5_SET(eqc, eqc, uar_page, dev->uar_page); + MLX5_SET(eqc, eqc, intr, MSI_VECTOR); + pas = MLX5_ADDR_OF(create_eq_in, in, pas); + VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->msi_eq_buf, pas), 0u); + MLX5_SET(eqc, eqc, log_page_size, 0); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->msi_eqn = MLX5_GET(create_eq_out, out, eq_number); + dev->msi_eq_cons_index = 0; + dev->have_msi_eq = true; + mlx5st_msi_eq_drain(dev); + + dev_dbg(device, + "Created MSI EQ: eqn=%u, %d entries (COMP), vector=%d\n", + dev->msi_eqn, MSI_EQ_NENT, MSI_VECTOR); +} + +static void mlx5st_destroy_msi_eq(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {}; + u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; + + MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); + MLX5_SET(destroy_eq_in, in, eq_number, dev->msi_eqn); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +/* + * HCA init / teardown + */ + +#define FW_INIT_TIMEOUT_MS 120000 +#define FW_INIT_WAIT_MS 200 + +static void mlx5st_wait_fw_init(struct mlx5st_device *dev) +{ + struct timespec start, now; + unsigned int elapsed; + + clock_gettime(CLOCK_MONOTONIC, &start); + while (MLX5_GET_MMIO(initial_seg, dev->init_seg, initializing)) { + usleep(FW_INIT_WAIT_MS * 1000); + clock_gettime(CLOCK_MONOTONIC, &now); + elapsed = (now.tv_sec - start.tv_sec) * 1000 + + (now.tv_nsec - start.tv_nsec) / 1000000; + if (elapsed > FW_INIT_TIMEOUT_MS) + VFIO_FAIL("FW init timeout after %d ms", elapsed); + } +} + +static void mlx5st_set_issi(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(set_issi_out)] = {}; + u32 in[MLX5_ST_SZ_DW(set_issi_in)] = {}; + + MLX5_SET(set_issi_in, in, opcode, MLX5_CMD_OP_SET_ISSI); + MLX5_SET(set_issi_in, in, current_issi, 1); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + dev_dbg(dev->device, "SET_ISSI: OK (issi=1)\n"); +} + +static void mlx5st_set_hca_caps(struct mlx5st_device *dev) +{ + u32 qout[MLX5_ST_SZ_DW(query_hca_cap_out)] = {}; + u32 qin[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; + u32 sout[MLX5_ST_SZ_DW(set_hca_cap_out)] = {}; + u32 sin[MLX5_ST_SZ_DW(set_hca_cap_in)] = {}; + struct mlx5_ifc_cmd_hca_cap_bits *set_hca_cap; + u32 max_checksum; + + /* Query max caps to learn cmdif_checksum support */ + MLX5_SET(query_hca_cap_in, qin, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); + MLX5_SET(query_hca_cap_in, qin, op_mod, + (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1)); + mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout)); + + max_checksum = MLX5_GET( + cmd_hca_cap, + MLX5_ADDR_OF(query_hca_cap_out, qout, capability), + cmdif_checksum); + + /* Query current caps as base for SET */ + memset(qout, 0, sizeof(qout)); + MLX5_SET(query_hca_cap_in, qin, op_mod, + (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) | + HCA_CAP_OPMOD_GET_CUR); + mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout)); + + set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, sin, capability); + memcpy(set_hca_cap, + MLX5_ADDR_OF(query_hca_cap_out, qout, capability), + MLX5_ST_SZ_BYTES(cmd_hca_cap)); + + MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, max_checksum); + MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, 0); + + MLX5_SET(set_hca_cap_in, sin, opcode, MLX5_CMD_OP_SET_HCA_CAP); + MLX5_SET(set_hca_cap_in, sin, op_mod, + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1); + + mlx5st_cmd_exec(dev, sin, sizeof(sin), sout, sizeof(sout)); + + dev->cmd_sig_enabled = max_checksum == 0x3; + dev_dbg(dev->device, "SET_HCA_CAP: OK (cmdif_checksum=%u)\n", + max_checksum); +} + +static void mlx5st_hca_init(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {}; + u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {}; + + mlx5st_wait_fw_init(dev); + dev_dbg(dev->device, "Firmware ready\n"); + + MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + dev_dbg(dev->device, "ENABLE_HCA: OK\n"); + + mlx5st_set_issi(dev); + mlx5st_fw_pages_satisfy(dev, 1); + + mlx5st_set_hca_caps(dev); + mlx5st_fw_pages_satisfy(dev, 0); + + memset(in, 0, sizeof(in)); + memset(out, 0, sizeof(out)); + MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + dev_dbg(dev->device, "INIT_HCA: OK\n"); + + /* + * Create EQ immediately after INIT_HCA so PAGE_REQUEST events + * are captured during all subsequent commands. + */ + mlx5st_alloc_uar(dev); + mlx5st_create_eq(dev); +} + +static void mlx5st_disable_hca(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {}; + u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {}; + + MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +static void mlx5st_fw_pages_reclaim(struct mlx5st_device *dev, u16 func_id) +{ + unsigned int npages = dev->fw_pages_given; + unsigned int total_claimed = 0; + + while (npages > 0) { + unsigned int batch = npages < MAX_FW_PAGES_PER_CMD ? + npages : + MAX_FW_PAGES_PER_CMD; + unsigned int outlen = + MLX5_ST_SZ_BYTES(manage_pages_out) + batch * 8; + unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); + unsigned int num_claimed; + unsigned int i; + void *in, *out; + u64 *iovas; + + in = calloc(1, inlen); + out = calloc(1, outlen); + iovas = calloc(batch, sizeof(u64)); + VFIO_ASSERT_NOT_NULL(in); + VFIO_ASSERT_NOT_NULL(out); + VFIO_ASSERT_NOT_NULL(iovas); + + MLX5_SET(manage_pages_in, in, opcode, + MLX5_CMD_OP_MANAGE_PAGES); + MLX5_SET(manage_pages_in, in, op_mod, + MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES); + MLX5_SET(manage_pages_in, in, function_id, func_id); + MLX5_SET(manage_pages_in, in, input_num_entries, batch); + + mlx5st_cmd_exec(dev, in, inlen, out, outlen); + + num_claimed = + MLX5_GET(manage_pages_out, out, output_num_entries); + for (i = 0; i < num_claimed; i++) + iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas, + i); + + mlx5st_fw_pages_free(dev, num_claimed, iovas); + total_claimed += num_claimed; + npages -= num_claimed; + + free(iovas); + free(in); + free(out); + + if (!num_claimed && !dev->fw_pages_given) + break; + if (!num_claimed) + VFIO_FAIL("MANAGE_PAGES RECLAIM: FW returned 0 but %d pages still given", + dev->fw_pages_given); + } + + dev_dbg(dev->device, + "MANAGE_PAGES RECLAIM: %d pages (%d still given)\n", + total_claimed, dev->fw_pages_given); +} + +static void mlx5st_hca_teardown(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {}; + u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {}; + + /* Drain async pages slot, then stop EQ processing */ + while (dev->pages_slot_in_use) { + if (!MLX5_GET_ONCE(cmd_queue_entry, dev->pages_cmd_lay, + ownership)) + mlx5st_pages_slot_complete(dev); + else + sched_yield(); + } + dev->have_eq = false; + + if (dev->eqn) { + mlx5st_destroy_eq(dev); + dev->eqn = 0; + } + if (dev->uar_page) { + mlx5st_dealloc_uar(dev); + dev->uar_page = 0; + } + + dev_dbg(dev->device, " hca_teardown: TEARDOWN_HCA\n"); + MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); + MLX5_SET(teardown_hca_in, in, profile, + MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + if (dev->fw_pages_given > 0) { + dev_dbg(dev->device, " hca_teardown: reclaim %d pages\n", + dev->fw_pages_given); + mlx5st_fw_pages_reclaim(dev, dev->fw_func_id); + } + + dev_dbg(dev->device, " hca_teardown: DISABLE_HCA\n"); + mlx5st_disable_hca(dev); +} + +/* + * Query capabilities + */ +static void mlx5st_query_fl_caps(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; + bool fl_roce_en, fl_roce_dis; + + /* Query RoCE capabilities */ + MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); + MLX5_SET(query_hca_cap_in, in, op_mod, + (MLX5_SET_HCA_CAP_OP_MOD_ROCE << 1) | HCA_CAP_OPMOD_GET_CUR); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + fl_roce_en = MLX5_GET(query_hca_cap_out, out, + capability.roce_cap.fl_rc_qp_when_roce_enabled); + fl_roce_dis = MLX5_GET(query_hca_cap_out, out, + capability.roce_cap.fl_rc_qp_when_roce_disabled); + + /* Also check general caps */ + memset(in, 0, sizeof(in)); + memset(out, 0, sizeof(out)); + MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); + MLX5_SET(query_hca_cap_in, in, op_mod, + (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) | + HCA_CAP_OPMOD_GET_CUR); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + fl_roce_dis |= + MLX5_GET(query_hca_cap_out, out, + capability.cmd_hca_cap.fl_rc_qp_when_roce_disabled); + + dev->fl_supported = fl_roce_en || fl_roce_dis; + dev->log_max_msg = MLX5_GET(query_hca_cap_out, out, + capability.cmd_hca_cap.log_max_msg); + dev_dbg(dev->device, + "HCA capabilities: fl_roce_enabled=%d fl_roce_disabled=%d log_max_msg=%u\n", + fl_roce_en, fl_roce_dis, dev->log_max_msg); + + VFIO_ASSERT_TRUE(dev->fl_supported, + "Force-loopback not supported on this device"); +} + +/* + * Resource allocation + */ + +static void mlx5st_alloc_pd(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; + u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; + + MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->pdn = MLX5_GET(alloc_pd_out, out, pd); + dev_dbg(dev->device, "Allocated PD pdn=%u\n", dev->pdn); +} + +static void mlx5st_dealloc_pd(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {}; + u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {}; + + MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD); + MLX5_SET(dealloc_pd_in, in, pd, dev->pdn); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +static void mlx5st_create_mkey(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(create_mkey_out)] = {}; + u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; + struct mlx5_ifc_mkc_bits *mkc; + + MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); + + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); + MLX5_SET(mkc, mkc, length64, 1); + MLX5_SET(mkc, mkc, pd, dev->pdn); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + MLX5_SET(mkc, mkc, lr, 1); + MLX5_SET(mkc, mkc, lw, 1); + MLX5_SET(mkc, mkc, rw, 1); + MLX5_SET(mkc, mkc, rr, 1); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->mkey_index = MLX5_GET(create_mkey_out, out, mkey_index); + dev->global_lkey = mlx5st_idx_to_mkey(dev->mkey_index); + dev->global_rkey = dev->global_lkey; + + dev_dbg(dev->device, "Created global PA-mode MKEY: lkey=0x%x\n", + dev->global_lkey); +} + +static void mlx5st_destroy_mkey(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {}; + u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; + + MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); + MLX5_SET(destroy_mkey_in, in, mkey_index, dev->mkey_index); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +/* + * CQ create/destroy + */ + +static void mlx5st_create_cq(struct mlx5st_device *dev) +{ + struct vfio_pci_device *device = dev->device; + u64 in[MLX5_ST_SZ_QW(create_cq_in) + 1] = {}; + u32 out[MLX5_ST_SZ_DW(create_cq_out)] = {}; + struct mlx5_ifc_cqc_bits *cqc; + unsigned int i; + __be64 *pas; + + /* Initialize CQEs before CREATE_CQ: opcode=0xF, owner=1 */ + for (i = 0; i < CQ_CQE_CNT; i++) { + struct mlx5st_cqe64 *cqe = &dev->cq_buf[i]; + + MLX5_SET(cqe64, cqe, opcode, 0xF); + MLX5_SET_ONCE(cqe64, cqe, owner, 1); + } + + MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ); + + cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); + MLX5_SET(cqc, cqc, log_cq_size, LOG_CQ_SIZE); + MLX5_SET(cqc, cqc, uar_page, dev->uar_page); + MLX5_SET(cqc, cqc, c_eqn_or_apu_element, dev->msi_eqn); + MLX5_SET(cqc, cqc, cqe_sz, 0); + pas = MLX5_ADDR_OF(create_cq_in, in, pas); + MLX5_SET(cqc, cqc, page_offset, mlx5st_fill_pas(device, dev->cq_buf, pas)); + MLX5_SET(cqc, cqc, log_page_size, 0); + MLX5_SET64(cqc, cqc, dbr_addr, to_iova(device, &dev->cq_dbrec)); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->cqn = MLX5_GET(create_cq_out, out, cqn); + dev->cq_ci = 0; + dev_dbg(device, "Created CQ: cqn=%u, %d entries\n", dev->cqn, + CQ_CQE_CNT); +} + +static void mlx5st_destroy_cq(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {}; + u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {}; + + MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ); + MLX5_SET(destroy_cq_in, in, cqn, dev->cqn); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +/* + * Arm CQ for event generation. The CQ event delivery state machine is + * single-shot: after generating one EQE the CQ enters "Fired" state and + * won't generate another until re-armed via ARM_NEXT. Both the CQ doorbell + * record and the UAR CQ doorbell register must be written. + */ +static void mlx5st_arm_cq(struct mlx5st_device *dev) +{ + u32 sn = dev->cq_arm_sn & 3; + u32 ci = dev->cq_ci & 0xffffff; + u64 doorbell; + + /* Update CQ doorbell record arm word */ + WRITE_ONCE(dev->cq_dbrec.send_counter, + cpu_to_be32(sn << 28 | ci)); + + /* Ring CQ doorbell register, iowrite has an internal dma_wmb() */ + doorbell = ((u64)(sn << 28 | ci) << 32) | dev->cqn; + iowrite64be(doorbell, + (u8 __iomem *)dev->uar_base + MLX5_CQ_DOORBELL_OFFSET); + + dev->cq_arm_sn++; +} + +/* + * QP create/destroy + */ + +static void mlx5st_create_qp(struct mlx5st_device *dev) +{ + struct vfio_pci_device *device = dev->device; + u64 in[MLX5_ST_SZ_QW(create_qp_in) + 1] = {}; + u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; + struct mlx5_ifc_qpc_bits *qpc; + __be64 *pas; + + MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); + + qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); + MLX5_SET(qpc, qpc, st, MLX5_QPC_ST_RC); + MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); + MLX5_SET(qpc, qpc, pd, dev->pdn); + MLX5_SET(qpc, qpc, uar_page, dev->uar_page); + MLX5_SET(qpc, qpc, cqn_snd, dev->cqn); + MLX5_SET(qpc, qpc, cqn_rcv, dev->cqn); + MLX5_SET(qpc, qpc, log_sq_size, LOG_SQ_SIZE); + MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg); + MLX5_SET(qpc, qpc, rq_type, 0x3); + MLX5_SET(qpc, qpc, ts_format, 1); + pas = MLX5_ADDR_OF(create_qp_in, in, pas); + MLX5_SET(qpc, qpc, page_offset, + mlx5st_fill_pas(device, dev->sq_buf, pas)); + MLX5_SET(qpc, qpc, log_page_size, 0); + MLX5_SET64(qpc, qpc, dbr_addr, to_iova(device, &dev->qp_dbrec)); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + + dev->qpn = MLX5_GET(create_qp_out, out, qpn); + dev->sq_pi = 0; + dev_dbg(device, "Created QP: qpn=%u, RC, sq=%d wqes\n", dev->qpn, + SQ_WQE_CNT); +} + +static void mlx5st_destroy_qp(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(destroy_qp_out)] = {}; + u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; + + MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); + MLX5_SET(destroy_qp_in, in, qpn, dev->qpn); + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +/* + * QP state transitions + */ + +static void mlx5st_qp_rst2init(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(rst2init_qp_out)] = {}; + u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; + struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc); + + MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP); + MLX5_SET(rst2init_qp_in, in, qpn, dev->qpn); + + MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); + MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); + MLX5_SET(qpc, qpc, rre, 1); + MLX5_SET(qpc, qpc, rwe, 1); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + dev_dbg(dev->device, "QP RST->INIT\n"); +} + +static void mlx5st_qp_init2rtr(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(init2rtr_qp_out)] = {}; + u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; + struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc); + + MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP); + MLX5_SET(init2rtr_qp_in, in, qpn, dev->qpn); + + MLX5_SET(qpc, qpc, mtu, 3); + MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg); + MLX5_SET(qpc, qpc, remote_qpn, dev->qpn); + MLX5_SET(qpc, qpc, min_rnr_nak, 12); + MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); + MLX5_SET(qpc, qpc, primary_address_path.fl, 1); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + dev_dbg(dev->device, "QP INIT->RTR (fl=1)\n"); +} + +static void mlx5st_qp_rtr2rts(struct mlx5st_device *dev) +{ + u32 out[MLX5_ST_SZ_DW(rtr2rts_qp_out)] = {}; + u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; + struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc); + + MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP); + MLX5_SET(rtr2rts_qp_in, in, qpn, dev->qpn); + + MLX5_SET(qpc, qpc, log_ack_req_freq, 0); + MLX5_SET(qpc, qpc, retry_count, 7); + MLX5_SET(qpc, qpc, rnr_retry, 7); + MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 14); + + mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + dev_dbg(dev->device, "QP RTR->RTS\n"); +} + +/* + * Post RDMA Write WQE + */ +static void mlx5st_post_rdma_write(struct mlx5st_device *dev, u64 src_addr, + u32 src_lkey, u64 dst_addr, u32 dst_rkey, + u32 length, bool signaled) +{ + struct mlx5st_send_wqe *wqe; + unsigned int idx; + + idx = dev->sq_pi % SQ_WQE_CNT; + wqe = &dev->sq_buf[idx]; + + memset(wqe, 0, sizeof(*wqe)); + MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, opcode, MLX5_OPCODE_RDMA_WRITE); + MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, wqe_index, dev->sq_pi); + MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, qp_or_sq, dev->qpn); + MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ds, MLX5_RDMA_WRITE_DS); + if (signaled) + MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ce, MLX5_WQE_CE_CQE_ALWAYS); + + MLX5_SET64(wqe_raddr_seg, &wqe->raddr, raddr, dst_addr); + MLX5_SET(wqe_raddr_seg, &wqe->raddr, rkey, dst_rkey); + + MLX5_SET(wqe_data_seg, &wqe->data, byte_count, length); + MLX5_SET(wqe_data_seg, &wqe->data, lkey, src_lkey); + MLX5_SET64(wqe_data_seg, &wqe->data, addr, src_addr); + + dev->sq_pi++; + + /* Ensure WQE is visible to device before doorbell record */ + dma_wmb(); + + WRITE_ONCE(dev->qp_dbrec.send_counter, + cpu_to_be32(dev->sq_pi & 0xffff)); + + /* + * Ring doorbell: write first 8 bytes of ctrl to UAR BF register, + * iowrite has an internal dma_wmb() so the doorbell record will be + * visible. + */ + iowrite64be(be64_to_cpu(*(__be64 *)wqe), + (u8 __iomem *)dev->uar_base + dev->uar_bf_offset); + dev->uar_bf_offset ^= MLX5_BF_SIZE; +} + +/* + * Poll CQ + */ +static int mlx5st_poll_cq_batch(struct mlx5st_device *dev, + unsigned int max_cqe) +{ + unsigned int polled = 0; + + while (polled < max_cqe) { + unsigned int idx = dev->cq_ci % CQ_CQE_CNT; + struct mlx5st_cqe64 *cqe = &dev->cq_buf[idx]; + u8 owner, opcode; + + owner = MLX5_GET_ONCE(cqe64, cqe, owner); + if (owner != ((dev->cq_ci >> LOG_CQ_SIZE) & 1)) + break; + + dma_rmb(); + + opcode = MLX5_GET(cqe64, cqe, opcode); + + dev->cq_ci++; + WRITE_ONCE(dev->cq_dbrec.recv_counter, + cpu_to_be32(dev->cq_ci & 0xffffff)); + + if (opcode == MLX5_CQE_REQ) { + dev->sq_ci = + (u16)(MLX5_GET(cqe64, cqe, wqe_counter) + 1); + polled++; + continue; + } + if (opcode == MLX5_CQE_REQ_ERR || + opcode == MLX5_CQE_RESP_ERR) { + dev_dbg(dev->device, + "CQE error: opcode=0x%x syndrome=0x%x vendor=0x%x\n", + opcode, + MLX5_GET(cqe64, cqe, error_syndrome.syndrome), + MLX5_GET(cqe64, cqe, + error_syndrome.vendor_error_syndrome)); + return -1; + } + dev_err(dev->device, "CQE unexpected opcode=0x%x\n", opcode); + return -1; + } + + return polled; +} + +static int mlx5st_poll_cq(struct mlx5st_device *dev, unsigned int timeout_ms) +{ + struct timespec start, now; + unsigned int elapsed; + int ret; + + clock_gettime(CLOCK_MONOTONIC, &start); + for (;;) { + ret = mlx5st_poll_cq_batch(dev, 1); + if (ret < 0) + return -1; + if (ret > 0) + return 0; + + if (dev->have_eq) + mlx5st_process_events(dev); + + clock_gettime(CLOCK_MONOTONIC, &now); + elapsed = (now.tv_sec - start.tv_sec) * 1000 + + (now.tv_nsec - start.tv_nsec) / 1000000; + if (elapsed > timeout_ms) { + dev_err(dev->device, "CQ poll timeout after %u ms\n", + timeout_ms); + return -1; + } + } +} + +/* + * Data path setup/teardown helpers + */ + +static void mlx5st_setup_datapath(struct mlx5st_device *dev) +{ + mlx5st_create_cq(dev); + mlx5st_create_qp(dev); + mlx5st_qp_rst2init(dev); + mlx5st_qp_init2rtr(dev); + mlx5st_qp_rtr2rts(dev); +} + +static void mlx5st_teardown_datapath(struct mlx5st_device *dev) +{ + if (dev->qpn) { + mlx5st_destroy_qp(dev); + dev->qpn = 0; + } + if (dev->cqn) { + mlx5st_destroy_cq(dev); + dev->cqn = 0; + } + dev->sq_pi = 0; + dev->sq_ci = 0; + dev->cq_arm_sn = 0; + memset(&dev->qp_dbrec, 0, sizeof(dev->qp_dbrec)); + memset(&dev->cq_dbrec, 0, sizeof(dev->cq_dbrec)); +} + +/* + * memcpy callbacks + */ + +#define MLX5ST_MEMCPY_TIMEOUT_MS 60000 + +static void mlx5st_memcpy_start(struct vfio_pci_device *device, + iova_t src, iova_t dst, u64 size, u64 count) +{ + struct mlx5st_device *dev = to_mlx5st(device); + u64 i; + + for (i = 0; i < count; i++) { + bool signaled = (i == count - 1); + + mlx5st_post_rdma_write(dev, src, dev->global_lkey, dst, + dev->global_rkey, size, signaled); + } +} + +static int mlx5st_memcpy_wait(struct vfio_pci_device *device) +{ + struct mlx5st_device *dev = to_mlx5st(device); + int ret; + + ret = mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS); + if (ret) { + /* + * CQE error puts the QP in error state. Rebuild the data path + * so subsequent operations can succeed. + */ + mlx5st_teardown_datapath(dev); + mlx5st_setup_datapath(dev); + } + return ret; +} + +/* + * send_msi callback — trigger CQE -> EQE -> MSI-X via a small RDMA Write. + * + * Both the CQ and MSI EQ use single-shot arming: the CQ must be armed so the + * CQE generates an EQE, and the MSI EQ must be armed so the EQE fires MSI-X. + */ +static void mlx5st_send_msi(struct vfio_pci_device *device) +{ + struct mlx5st_device *dev = to_mlx5st(device); + + /* Drain accumulated MSI EQ events and re-arm for next interrupt */ + mlx5st_msi_eq_drain(dev); + + /* Arm CQ so the next CQE generates an EQE on the MSI EQ */ + mlx5st_arm_cq(dev); + + /* Post a signaled RDMA Write to trigger CQE -> EQE -> MSI-X */ + mlx5st_post_rdma_write(dev, + to_iova(device, &dev->send_msi_src), + dev->global_lkey, + to_iova(device, &dev->send_msi_dst), + dev->global_rkey, + sizeof(dev->send_msi_src), true); + + /* Consume the CQE to avoid stale completions */ + VFIO_ASSERT_EQ(mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS), 0); +} + +/* + * Driver ops callbacks + */ + +static void mlx5st_init(struct vfio_pci_device *device) +{ + struct mlx5st_device *dev = to_mlx5st(device); + iova_t iova_align = + device->driver.region.iova % __alignof__(struct mlx5st_device); + + VFIO_ASSERT_GE(device->driver.region.size, sizeof(*dev)); + VFIO_ASSERT_EQ(iova_align, 0); + memset(dev, 0, sizeof(*dev)); + + dev->device = device; + dev->bar0 = device->bars[0].vaddr; + dev->init_seg = dev->bar0; + + vfio_pci_config_writew(device, PCI_COMMAND, + vfio_pci_config_readw(device, PCI_COMMAND) | + PCI_COMMAND_MASTER); + + mlx5st_wait_fw_init(dev); + + mlx5st_cmd_init(dev); + mlx5st_hca_init(dev); + mlx5st_query_fl_caps(dev); + mlx5st_alloc_pd(dev); + mlx5st_create_mkey(dev); + + /* MSI EQ must be created before CQ so CQ can reference its eqn */ + mlx5st_create_msi_eq(dev); + mlx5st_setup_datapath(dev); + + vfio_pci_msix_enable(device, MSI_VECTOR, 1); + device->driver.msi = MSI_VECTOR; + + device->driver.max_memcpy_size = 1ULL << dev->log_max_msg; + device->driver.max_memcpy_count = SQ_WQE_CNT - 1; + + dev_dbg(device, "mlx5 driver initialized\n"); +} + +static void mlx5st_remove(struct vfio_pci_device *device) +{ + struct mlx5st_device *dev = to_mlx5st(device); + + vfio_pci_msix_disable(device); + mlx5st_teardown_datapath(dev); + + if (dev->have_msi_eq) { + mlx5st_destroy_msi_eq(dev); + dev->have_msi_eq = false; + } + + dev_dbg(device, "teardown: destroy_mkey\n"); + if (dev->mkey_index) { + mlx5st_destroy_mkey(dev); + dev->mkey_index = 0; + } + + dev_dbg(device, "teardown: dealloc_pd\n"); + if (dev->pdn) { + mlx5st_dealloc_pd(dev); + dev->pdn = 0; + } + + dev_dbg(device, "teardown: hca_teardown\n"); + mlx5st_hca_teardown(dev); + + vfio_pci_config_writew(device, PCI_COMMAND, + vfio_pci_config_readw(device, PCI_COMMAND) & + ~(u16)PCI_COMMAND_MASTER); + + dev_dbg(device, "Teardown complete\n"); +} + +struct vfio_pci_driver_ops mlx5st_ops = { + .name = "mlx5", + .region_size = sizeof(struct mlx5st_device), + .probe = mlx5st_probe, + .init = mlx5st_init, + .remove = mlx5st_remove, + .memcpy_start = mlx5st_memcpy_start, + .memcpy_wait = mlx5st_memcpy_wait, + .send_msi = mlx5st_send_msi, +};
diff --git a/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_hw.h b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_hw.h new file mode 100644 index 0000000..2c451e4 --- /dev/null +++ b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_hw.h
@@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * mlx5 VFIO selftest driver - HW definitions + * + * Typed wrappers, constants, and helpers for programming mlx5 hardware + * via the VFIO selftest framework. Most HW constants and all MLX5_SET/GET + * macros come from the kernel headers (mlx5_ifc.h, mlx5_ifc_macros.h). + */ +#ifndef SELFTESTS_VFIO_MLX5_HW_H +#define SELFTESTS_VFIO_MLX5_HW_H + +#include <linux/io.h> +#include <linux/build_bug.h> +#include <vdso/bits.h> + +#include "mlx5_ifc.h" +#include "mlx5_ifc_macros.h" + +/* + * Typed HW object wrappers for driver region arrays. + * + * The IFC _bits structs have sizeof == num_bits (not bytes), so they cannot + * be used as array elements. These wrappers provide byte-sized types. + */ +#define MLX5ST_MAKE_DATA32(name) \ + struct mlx5st_##name { \ + u32 data[MLX5_ST_SZ_DW(name)]; \ + } +#define MLX5ST_MAKE_DATA64(name) \ + struct mlx5st_##name { \ + u64 data[MLX5_ST_SZ_QW(name)]; \ + } + +MLX5ST_MAKE_DATA32(initial_seg); +MLX5ST_MAKE_DATA64(cmd_queue_entry); +MLX5ST_MAKE_DATA64(cmd_if_box); +MLX5ST_MAKE_DATA64(wqe_ctrl_seg); +MLX5ST_MAKE_DATA64(wqe_raddr_seg); +MLX5ST_MAKE_DATA64(wqe_data_seg); +MLX5ST_MAKE_DATA64(cqe64) __aligned(64); +MLX5ST_MAKE_DATA64(eqe); + +/* + * Mailbox blocks: 512 data + 64 header = 576 bytes, but the + * next_pointer field stores bits [31:10], requiring 1024-byte alignment. + */ +#define CMD_MBOX_SIZE (2 * MLX5_HW_PAGE_SIZE) +#define CMD_MBOX_STRIDE 1024 +#define CMD_MBOX_NENT (CMD_MBOX_SIZE / CMD_MBOX_STRIDE) +/* Stride-aligned mailbox entry — block + padding to 1024 bytes */ +struct mlx5st_mbox_entry { + struct mlx5st_cmd_if_box block; +} __aligned(CMD_MBOX_STRIDE); + +#define MLX5_CMD_INLINE_SZ \ + MLX5_FLD_SZ_BYTES(cmd_queue_entry, command_input_inline_data) + +/* Command interface mailbox block (512 data + 64 header) */ +#define MLX5_CMD_DATA_BLOCK_SIZE MLX5_FLD_SZ_BYTES(cmd_if_box, mailbox_data) + +/* RDMA Write WQE — one basic block: ctrl + raddr + data + padding */ +struct mlx5st_send_wqe { + struct mlx5st_wqe_ctrl_seg ctrl; + struct mlx5st_wqe_raddr_seg raddr; + struct mlx5st_wqe_data_seg data; +} __aligned(64); +static_assert(sizeof(struct mlx5st_send_wqe) == 64, + "send WQE segments must fit in one BB"); + +/* DS = number of 16-byte segments in the WQE (ctrl + raddr + data) */ +#define MLX5_RDMA_WRITE_DS 3 + +/* Doorbell record — two __be32 in a 64-byte aligned pair */ +struct mlx5st_dbrec { + __be32 recv_counter; + __be32 send_counter; +} __aligned(64); + +/* UAR BlueFlame buffer offsets within a UAR page */ +#define MLX5_BF_OFFSET 0x800 +#define MLX5_BF_SIZE 0x100 + +/* CQ doorbell offset within UAR page */ +#define MLX5_CQ_DOORBELL_OFFSET 0x20 + +/* EQ doorbell offset within UAR page */ +#define MLX5_EQ_DOORBELL_OFFSET 0x40 + +#define MLX5_HW_PAGE_SIZE 4096 + +/* + * Test parameters + */ +#define SQ_WQE_CNT 16 +#define LOG_SQ_SIZE 4 +#define CQ_CQE_CNT 16 +#define LOG_CQ_SIZE 4 +#define EQ_NENT 64 +#define LOG_EQ_SIZE 6 +#define MSI_EQ_NENT 16 +#define LOG_MSI_EQ_SIZE 4 +#define MSI_VECTOR 0 + +#define MAX_FW_PAGES 8192 +#define MAX_FW_PAGES_PER_CMD 512 + +#define MLX5_CMD_TIMEOUT_MS 5000 + +static inline u32 mlx5st_idx_to_mkey(u32 mkey_idx) +{ + return mkey_idx << 8; +} + +#endif /* SELFTESTS_VFIO_MLX5_HW_H */
diff --git a/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc.h b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc.h new file mode 120000 index 0000000..7dcbb79 --- /dev/null +++ b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc.h
@@ -0,0 +1 @@ +../../../../../../../include/linux/mlx5/mlx5_ifc.h \ No newline at end of file
diff --git a/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_fpga.h b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_fpga.h new file mode 120000 index 0000000..865d99e --- /dev/null +++ b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_fpga.h
@@ -0,0 +1 @@ +../../../../../../../include/linux/mlx5/mlx5_ifc_fpga.h \ No newline at end of file
diff --git a/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_macros.h b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_macros.h new file mode 120000 index 0000000..97408c2 --- /dev/null +++ b/tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_macros.h
@@ -0,0 +1 @@ +../../../../../../../include/linux/mlx5/mlx5_ifc_macros.h \ No newline at end of file
diff --git a/tools/testing/selftests/vfio/lib/drivers/nv_falcon/nv_falcon.c b/tools/testing/selftests/vfio/lib/drivers/nv_falcon/nv_falcon.c index c08aa81..6958877 100644 --- a/tools/testing/selftests/vfio/lib/drivers/nv_falcon/nv_falcon.c +++ b/tools/testing/selftests/vfio/lib/drivers/nv_falcon/nv_falcon.c
@@ -775,6 +775,7 @@ static int nv_falcon_memcpy_wait(struct vfio_pci_device *device) const struct vfio_pci_driver_ops nv_falcon_ops = { .name = "nv_falcon", + .region_size = sizeof(struct gpu_device), .probe = nv_falcon_probe, .init = nv_falcon_init, .remove = nv_falcon_remove,
diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h index e19bd94..007adc3 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h
@@ -40,6 +40,17 @@ struct vfio_pci_device { struct vfio_pci_device *vfio_pci_device_alloc(const char *bdf, struct iommu *iommu); void vfio_pci_device_free(struct vfio_pci_device *device); + +#ifdef DEBUG +#define dev_dbg dev_info +#else +#define dev_dbg(_dev, _fmt, ...) \ + do { \ + if (0) \ + dev_info(_dev, _fmt, ##__VA_ARGS__); \ + } while (0) +#endif + struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu); void vfio_pci_device_cleanup(struct vfio_pci_device *device);
diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_driver.h b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_driver.h index e5ada20..547369c 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_driver.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_driver.h
@@ -9,6 +9,12 @@ struct vfio_pci_device; struct vfio_pci_driver_ops { const char *name; + /* + * Size of the driver's state structure overlaid on + * device->driver.region.vaddr + */ + u64 region_size; + /** * @probe() - Check if the driver supports the given device. *
diff --git a/tools/testing/selftests/vfio/lib/iova_allocator.c b/tools/testing/selftests/vfio/lib/iova_allocator.c index 4a660f6..a8e5815 100644 --- a/tools/testing/selftests/vfio/lib/iova_allocator.c +++ b/tools/testing/selftests/vfio/lib/iova_allocator.c
@@ -13,8 +13,10 @@ #include <linux/iommufd.h> #include <linux/limits.h> +#include <linux/log2.h> #include <linux/mman.h> #include <linux/overflow.h> +#include <linux/sizes.h> #include <linux/types.h> #include <linux/vfio.h> @@ -50,7 +52,10 @@ void iova_allocator_cleanup(struct iova_allocator *allocator) iova_t iova_allocator_alloc(struct iova_allocator *allocator, size_t size) { VFIO_ASSERT_GT(size, 0, "Invalid size arg, zero\n"); - VFIO_ASSERT_EQ(size & (size - 1), 0, "Invalid size arg, non-power-of-2\n"); + + VFIO_ASSERT_LE(size, rounddown_pow_of_two(SIZE_MAX), + "Invalid size arg, too large (%zu)\n", size); + size = roundup_pow_of_two(size); for (;;) { struct iommu_iova_range *range;
diff --git a/tools/testing/selftests/vfio/lib/libvfio.mk b/tools/testing/selftests/vfio/lib/libvfio.mk index bcfa74a..eff9582 100644 --- a/tools/testing/selftests/vfio/lib/libvfio.mk +++ b/tools/testing/selftests/vfio/lib/libvfio.mk
@@ -15,6 +15,7 @@ LIBVFIO_C += drivers/dsa/dsa.c endif +LIBVFIO_C += drivers/mlx5/mlx5.c LIBVFIO_C += drivers/nv_falcon/nv_falcon.c LIBVFIO_C += drivers/igb/igb.c
diff --git a/tools/testing/selftests/vfio/lib/sysfs.c b/tools/testing/selftests/vfio/lib/sysfs.c index 98a46a2..d88e348 100644 --- a/tools/testing/selftests/vfio/lib/sysfs.c +++ b/tools/testing/selftests/vfio/lib/sysfs.c
@@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include <fcntl.h> +#include <libgen.h> #include <unistd.h> #include <stdlib.h> #include <string.h>
diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_driver.c b/tools/testing/selftests/vfio/lib/vfio_pci_driver.c index 5e65434..b31abfe 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_driver.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_driver.c
@@ -6,6 +6,7 @@ extern struct vfio_pci_driver_ops dsa_ops; extern struct vfio_pci_driver_ops ioat_ops; #endif +extern struct vfio_pci_driver_ops mlx5st_ops; extern struct vfio_pci_driver_ops nv_falcon_ops; extern struct vfio_pci_driver_ops igb_ops; @@ -14,6 +15,7 @@ static struct vfio_pci_driver_ops *driver_ops[] = { &dsa_ops, &ioat_ops, #endif + &mlx5st_ops, &nv_falcon_ops, &igb_ops, }; @@ -32,6 +34,10 @@ void vfio_pci_driver_probe(struct vfio_pci_device *device) continue; device->driver.ops = ops; + + VFIO_ASSERT_NE(ops->region_size, 0); + device->driver.region.size = + round_up(ops->region_size, getpagesize()); } }
diff --git a/tools/testing/selftests/vfio/settings b/tools/testing/selftests/vfio/settings new file mode 100644 index 0000000..c9abf1c --- /dev/null +++ b/tools/testing/selftests/vfio/settings
@@ -0,0 +1,5 @@ +# Extend kselftest default 45s timeout based on worst case test runtime, +# currently vfio:vfio_pci_driver_test running on physical NIC, battling +# SMIs due to invoked firmware-first error handling from bad DMAs and +# device wedging. Observed worst case ~450s, extend to 600s. +timeout=600
diff --git a/tools/testing/selftests/vfio/vfio_pci_driver_test.c b/tools/testing/selftests/vfio/vfio_pci_driver_test.c index 761bf11..9538a3a 100644 --- a/tools/testing/selftests/vfio/vfio_pci_driver_test.c +++ b/tools/testing/selftests/vfio/vfio_pci_driver_test.c
@@ -87,7 +87,8 @@ FIXTURE_SETUP(vfio_pci_driver_test) driver = &self->device->driver; region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G); - region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M); + region_setup(self->iommu, self->iova_allocator, &driver->region, + driver->region.size); /* Any IOVA that doesn't overlap memcpy_region and driver->region. */ self->unmapped_iova = iova_allocator_alloc(self->iova_allocator, SZ_1G); @@ -190,7 +191,7 @@ TEST_F(vfio_pci_driver_test, send_msi) ASSERT_EQ(1, value); } -TEST_F(vfio_pci_driver_test, mix_and_match) +TEST_F_TIMEOUT(vfio_pci_driver_test, mix_and_match, 90) { u64 value; int i;
diff --git a/tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c b/tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c index 19d657d..f3a018b 100644 --- a/tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c +++ b/tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c
@@ -157,6 +157,41 @@ TEST_F(vfio_pci_sriov_uapi_test, override_token) ASSERT_COND_VF_CREATION(ret); } +TEST(failed_second_open_does_not_clobber_token) +{ + struct vfio_pci_device *pf = NULL, *pf_second_fd = NULL, *vf = NULL; + struct iommu *iommu; + int ret; + + iommu = iommu_init(MODE_IOMMUFD); + + /* Create and bind PF using UUID_1 */ + ret = device_init(pf_bdf, iommu, UUID_1, &pf); + ASSERT_EQ(ret, 0); + + /* + * Attempt to open the same PF again and bind it with a *different* + * token (UUID_2). This must fail with -EBUSY because the cdev path + * only supports a single open per device. Enforce it with EXPECT_EQ() + * so the clobber assertion below still runs if the errno differs. + */ + ret = device_init(pf_bdf, iommu, UUID_2, &pf_second_fd); + EXPECT_EQ(ret, -EBUSY); + + /* + * Attempt to initialize a VF using the original PF token (UUID_1). + * If the failed open above clobbered the PF's token (i.e. updated it to + * UUID_2), this VF initialization will fail. + */ + ret = device_init(vf_bdf, iommu, UUID_1, &vf); + ASSERT_EQ(ret, 0); + + device_cleanup(vf); + device_cleanup(pf_second_fd); + device_cleanup(pf); + iommu_cleanup(iommu); +} + static void vf_teardown(void) { /*
diff --git a/tools/tracing/rtla/src/cli.c b/tools/tracing/rtla/src/cli.c index fb8c972..8c02b94 100644 --- a/tools/tracing/rtla/src/cli.c +++ b/tools/tracing/rtla/src/cli.c
@@ -78,6 +78,7 @@ struct common_params *osnoise_top_parse_args(int argc, char **argv) RTLA_OPT_STOP_TOTAL('S', "stop-total", "total sample"), OSNOISE_OPT_THRESHOLD, RTLA_OPT_TRACE_OUTPUT("osnoise", opt_osnoise_trace_output_cb), + OSNOISE_OPT_IPI, OPT_GROUP("Event Configuration:"), RTLA_OPT_EVENT,
diff --git a/tools/tracing/rtla/src/cli_p.h b/tools/tracing/rtla/src/cli_p.h index 661240d..7e3a713 100644 --- a/tools/tracing/rtla/src/cli_p.h +++ b/tools/tracing/rtla/src/cli_p.h
@@ -394,8 +394,10 @@ static int opt_event_cb(const struct option *opt, const char *arg, int unset) if (!tevent) fatal("Error alloc trace event"); - if (*events) + if (*events) { tevent->next = *events; + (*events)->prev = tevent; + } *events = tevent; return 0; @@ -497,6 +499,9 @@ static int opt_filter_cb(const struct option *opt, const char *arg, int unset) #define OSNOISE_OPT_THRESHOLD RTLA_OPT_LLONG('T', "threshold", ¶ms->threshold, "us", \ "the minimum delta to be considered a noise") +#define OSNOISE_OPT_IPI OPT_BOOLEAN(0, "ipi", ¶ms->common.ipi, \ + "track sources of IPIs") + /* * Callback functions for command line options for osnoise tools */
diff --git a/tools/tracing/rtla/src/common.c b/tools/tracing/rtla/src/common.c index 8c7f5e7..ac4172b 100644 --- a/tools/tracing/rtla/src/common.c +++ b/tools/tracing/rtla/src/common.c
@@ -6,7 +6,6 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> -#include <sys/sysinfo.h> #include "common.h" @@ -63,11 +62,22 @@ static void unset_signals(struct common_params *params) int common_apply_config(struct osnoise_tool *tool, struct common_params *params) { - int retval, i; + int retval, i, tracer_cpus; if (!params->sleep_time) params->sleep_time = 1; + tracer_cpus = osnoise_get_cpu_count(); + if (tracer_cpus < 0) { + err_msg("Failed to get tracer cpu count from osnoise/cpus\n"); + goto out_err; + } + if (tracer_cpus != nr_cpus) { + err_msg("Kernel reports %d cpus but rtla sees %d\n", + tracer_cpus, nr_cpus); + goto out_err; + } + retval = osnoise_set_cpus(tool->context, params->cpus ? params->cpus : "all"); if (retval) { err_msg("Failed to apply CPUs config\n"); @@ -155,7 +165,20 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[]) bool stopped; int retval; - nr_cpus = get_nprocs_conf(); + nr_cpus = get_possible_cpus(); + if (nr_cpus == -1) { + err_msg("Could not read number of possible cpus\n"); + goto out_exit; + } + if (nr_cpus == -2) { + err_msg("Could not parse number of possible cpus\n"); + goto out_exit; + } + if (nr_cpus == -3) { + err_msg("Unsupported non-contiguous or non-zero-based CPU topology\n"); + goto out_exit; + } + params = ops->parse_args(argc, argv); if (!params) exit(1); @@ -205,7 +228,7 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[]) if (params->threshold_actions.present[ACTION_TRACE_OUTPUT] || params->end_actions.present[ACTION_TRACE_OUTPUT]) { - tool->record = osnoise_init_trace_tool(ops->tracer); + tool->record = osnoise_init_trace_tool(params, ops->tracer); if (!tool->record) { err_msg("Failed to enable the trace instance\n"); goto out_free;
diff --git a/tools/tracing/rtla/src/common.h b/tools/tracing/rtla/src/common.h index 04b287a..ec2f6f8 100644 --- a/tools/tracing/rtla/src/common.h +++ b/tools/tracing/rtla/src/common.h
@@ -7,8 +7,41 @@ #include "utils.h" /* + * OSNOISE_LL_OPTIONS - list of long long options backed by tracefs files. + * OSNOISE_LL_OPTION(field_name, tracefs_path) + * + * OSNOISE_FLAG_OPTIONS - list of boolean options backed by osnoise/options. + * OSNOISE_FLAG_OPTION(field_name, option_string) + * + * These X-macro lists are invoked in four places: + * - struct osnoise_context field declarations (common.h), + * - function declarations for osnoise_set_<opt>/osnoise_restore_<opt> (common.h), + * - function definitions for set/restore/put (osnoise.c), + * - context initialization and teardown in osnoise_context_alloc() and + * osnoise_put_context() (osnoise.c). + */ +#define OSNOISE_LL_OPTIONS \ + OSNOISE_LL_OPTION(stop_us, "osnoise/stop_tracing_us") \ + OSNOISE_LL_OPTION(stop_total_us, "osnoise/stop_tracing_total_us") \ + OSNOISE_LL_OPTION(print_stack, "osnoise/print_stack") \ + OSNOISE_LL_OPTION(tracing_thresh, "tracing_thresh") \ + OSNOISE_LL_OPTION(timerlat_period_us, "osnoise/timerlat_period_us") \ + OSNOISE_LL_OPTION(timerlat_align_us, "osnoise/timerlat_align_us") + +#define OSNOISE_FLAG_OPTIONS \ + OSNOISE_FLAG_OPTION(irq_disable, "OSNOISE_IRQ_DISABLE") \ + OSNOISE_FLAG_OPTION(workload, "OSNOISE_WORKLOAD") \ + OSNOISE_FLAG_OPTION(timerlat_align, "TIMERLAT_ALIGN") + +/* * osnoise_context - read, store, write, restore osnoise configs. */ +#define OSNOISE_LL_OPTION(name, path) \ + long long orig_##name; \ + long long name; +#define OSNOISE_FLAG_OPTION(name, option_str) \ + int orig_opt_##name; \ + int opt_##name; struct osnoise_context { int flags; int ref; @@ -16,50 +49,17 @@ struct osnoise_context { char *curr_cpus; char *orig_cpus; - /* 0 as init value */ - unsigned long long orig_runtime_us; - unsigned long long runtime_us; + long long orig_runtime_us; + long long runtime_us; - /* 0 as init value */ - unsigned long long orig_period_us; - unsigned long long period_us; + long long orig_period_us; + long long period_us; - /* 0 as init value */ - long long orig_timerlat_period_us; - long long timerlat_period_us; - - /* 0 as init value */ - long long orig_tracing_thresh; - long long tracing_thresh; - - /* -1 as init value because 0 is disabled */ - long long orig_stop_us; - long long stop_us; - - /* -1 as init value because 0 is disabled */ - long long orig_stop_total_us; - long long stop_total_us; - - /* -1 as init value because 0 is disabled */ - long long orig_print_stack; - long long print_stack; - - /* -1 as init value because 0 is off */ - int orig_opt_irq_disable; - int opt_irq_disable; - - /* -1 as init value because 0 is off */ - int orig_opt_workload; - int opt_workload; - - /* -1 as init value because 0 is off */ - int orig_opt_timerlat_align; - int opt_timerlat_align; - - /* 0 as init value */ - unsigned long long orig_timerlat_align_us; - unsigned long long timerlat_align_us; + OSNOISE_LL_OPTIONS + OSNOISE_FLAG_OPTIONS }; +#undef OSNOISE_LL_OPTION +#undef OSNOISE_FLAG_OPTION extern volatile int stop_tracing; @@ -108,14 +108,13 @@ struct common_params { bool kernel_workload; bool user_data; bool aa_only; + bool ipi; struct actions threshold_actions; struct actions end_actions; struct timerlat_u_params user; }; -extern int nr_cpus; - #define for_each_monitored_cpu(cpu, common) \ for (cpu = 0; cpu < nr_cpus; cpu++) \ if (!(common)->cpus || CPU_ISSET(cpu, &(common)->monitored_cpus)) @@ -170,18 +169,25 @@ should_continue_tracing(const struct common_params *params) int common_threshold_handler(const struct osnoise_tool *tool); +int osnoise_get_cpu_count(void); int osnoise_set_cpus(struct osnoise_context *context, char *cpus); void osnoise_restore_cpus(struct osnoise_context *context); -int osnoise_set_workload(struct osnoise_context *context, bool onoff); +#define OSNOISE_LL_OPTION(name, path) \ + int osnoise_set_##name(struct osnoise_context *context, long long name); \ + void osnoise_restore_##name(struct osnoise_context *context); +#define OSNOISE_FLAG_OPTION(name, option_str) \ + int osnoise_set_##name(struct osnoise_context *context, bool onoff); \ + void osnoise_restore_##name(struct osnoise_context *context); +OSNOISE_LL_OPTIONS +OSNOISE_FLAG_OPTIONS +#undef OSNOISE_LL_OPTION +#undef OSNOISE_FLAG_OPTION void osnoise_destroy_tool(struct osnoise_tool *top); struct osnoise_tool *osnoise_init_tool(char *tool_name); -struct osnoise_tool *osnoise_init_trace_tool(const char *tracer); +struct osnoise_tool *osnoise_init_trace_tool(struct common_params *params, const char *tracer); bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record); -int osnoise_set_stop_us(struct osnoise_context *context, long long stop_us); -int osnoise_set_stop_total_us(struct osnoise_context *context, - long long stop_total_us); int common_apply_config(struct osnoise_tool *tool, struct common_params *params); int top_main_loop(struct osnoise_tool *tool);
diff --git a/tools/tracing/rtla/src/osnoise.c b/tools/tracing/rtla/src/osnoise.c index 4ff5dad..19b22a6 100644 --- a/tools/tracing/rtla/src/osnoise.c +++ b/tools/tracing/rtla/src/osnoise.c
@@ -125,15 +125,83 @@ void osnoise_put_cpus(struct osnoise_context *context) } /* - * osnoise_read_ll_config - read a long long value from a config + * osnoise_get_cpu_count - get the number of CPUs as seen by the osnoise tracer * - * returns -1 on error. + * Write "all" to osnoise/cpus and read back the real kernel-side list + * of all possible cpus. Restore the original value afterwards. + * + * Returns the number of CPUs, or -1 on error. */ -static long long osnoise_read_ll_config(char *rel_path) +int osnoise_get_cpu_count(void) +{ + char *orig, *readback; + int max_cpu; + + orig = tracefs_instance_file_read(NULL, "osnoise/cpus", NULL); + if (!orig) + return -1; + + if (tracefs_instance_file_write(NULL, "osnoise/cpus", "all\n") < 0) { + free(orig); + return -1; + } + + readback = tracefs_instance_file_read(NULL, "osnoise/cpus", NULL); + if (!readback) { + tracefs_instance_file_write(NULL, "osnoise/cpus", orig); + free(orig); + return -1; + } + + max_cpu = get_max_cpu_from_list(readback); + free(readback); + + if (tracefs_instance_file_write(NULL, "osnoise/cpus", orig) < 0) { + free(orig); + return -1; + } + free(orig); + + if (max_cpu < 0) + return -1; + + return max_cpu + 1; +} + +/** + * osnoise_read_ll_config - read a long long value from a tracefs config + * @rel_path: tracefs-relative path to the config file + * @set_value: pointer to the cached value set by RTLA, or -1 if unset + * @orig_value: pointer to the cached original value read from tracefs, or -1 if unread + * + * Returns the current effective value for the config at @rel_path. If RTLA has + * already written a value (@set_value != -1), that value is returned. If the + * original has already been read (@orig_value != -1), that cached value is + * returned. Otherwise, reads the value from tracefs, caches it in @orig_value + * (so it can later be restored), and returns it. + * + * This is the shared read primitive used by both the manually implemented + * get functions (osnoise_get_runtime, osnoise_get_period) and the + * OSNOISE_LL_OPTION-generated set/restore/put functions. + * + * Returns the config value on success, -1 on error. + */ +static long long osnoise_read_ll_config(char *rel_path, + long long *set_value, + long long *orig_value) { long long retval; char *buffer; + if (*set_value != -1) + /* option has been set by RTLA already */ + return *set_value; + + if (*orig_value != -1) + /* RTLA has already read the option */ + return *orig_value; + + /* current value is not known to RTLA yet, read it from tracefs */ buffer = tracefs_instance_file_read(NULL, rel_path, NULL); if (!buffer) return -1; @@ -145,25 +213,98 @@ static long long osnoise_read_ll_config(char *rel_path) free(buffer); + if (retval < 0) + goto out_err; + + /* save the value and return it */ + *orig_value = retval; return retval; + +out_err: + return -1; } -/* - * osnoise_write_ll_config - write a long long value to a config in rel_path +/** + * osnoise_write_ll_config - write a long long value to a tracefs config + * @rel_path: tracefs-relative path to the config file + * @value: the value to write + * @set_value: pointer to the cached value set by RTLA, or -1 if unset + * @orig_value: pointer to the cached original value read from tracefs, or -1 if unread * - * returns -1 on error. + * Writes @value to the config at @rel_path. Before writing, calls + * osnoise_read_ll_config() to ensure the original value is cached in + * @orig_value (enabling later restoration). On successful write, records + * the new value in @set_value. + * + * This is the shared write primitive used by both the manually implemented + * write functions (__osnoise_write_runtime, __osnoise_write_period) and the + * OSNOISE_LL_OPTION-generated set/restore/put functions. + * + * Returns 0 on success, -1 on read error (option likely unknown to kernel), + * or -2 on write error. */ -static long long osnoise_write_ll_config(char *rel_path, long long value) +static int osnoise_write_ll_config(char *rel_path, + long long value, + long long *set_value, + long long *orig_value) { - char buffer[BUFF_U64_STR_SIZE]; + long long curr = osnoise_read_ll_config(rel_path, set_value, orig_value); long long retval; + char buffer[BUFF_U64_STR_SIZE]; + + if (curr == -1) + /* read failed, option likely unknown to kernel */ + return -1; snprintf(buffer, sizeof(buffer), "%lld\n", value); debug_msg("setting %s to %lld\n", rel_path, value); retval = tracefs_instance_file_write(NULL, rel_path, buffer); - return retval; + + if (retval < 0) + /* write failed, hard error */ + return -2; + + /* record the set value and return success */ + *set_value = value; + return 0; +} + +/** + * osnoise_restore_ll_config - restore a long long config to its original value + * @rel_path: tracefs-relative path to the config file + * @set_value: pointer to the cached value set by RTLA, or -1 if unset + * @orig_value: pointer to the cached original value read from tracefs, or -1 if unread + * + * Restores the config at @rel_path to the value cached in @orig_value (which + * was saved by a prior osnoise_read_ll_config() or osnoise_write_ll_config() + * call). If the original was never read, or if the current set value already + * matches the original, no write is performed. After restoring, clears + * @set_value to -1 to indicate RTLA no longer overrides this config. + * + * This is the shared restore primitive used by both the manually implemented + * restore function (osnoise_restore_runtime_period) and the + * OSNOISE_LL_OPTION-generated restore/put functions. + */ +static void osnoise_restore_ll_config(char *rel_path, + long long *set_value, + long long *orig_value) +{ + int retval; + + if (*orig_value == -1) + return; + + if (*orig_value == *set_value) + goto out_done; + + retval = osnoise_write_ll_config(rel_path, *orig_value, set_value, orig_value); + if (retval < 0) + err_msg("Could not restore original value for %s\n", rel_path); + +out_done: + *set_value = -1; } /* @@ -171,25 +312,11 @@ static long long osnoise_write_ll_config(char *rel_path, long long value) * * It also saves the value to be restored. */ -unsigned long long osnoise_get_runtime(struct osnoise_context *context) +long long osnoise_get_runtime(struct osnoise_context *context) { - long long runtime_us; - - if (context->runtime_us != OSNOISE_TIME_INIT_VAL) - return context->runtime_us; - - if (context->orig_runtime_us != OSNOISE_TIME_INIT_VAL) - return context->orig_runtime_us; - - runtime_us = osnoise_read_ll_config("osnoise/runtime_us"); - if (runtime_us < 0) - goto out_err; - - context->orig_runtime_us = runtime_us; - return runtime_us; - -out_err: - return OSNOISE_TIME_INIT_VAL; + return osnoise_read_ll_config("osnoise/runtime_us", + &context->runtime_us, + &context->orig_runtime_us); } /* @@ -197,57 +324,29 @@ unsigned long long osnoise_get_runtime(struct osnoise_context *context) * * It also saves the value to be restored. */ -unsigned long long osnoise_get_period(struct osnoise_context *context) +long long osnoise_get_period(struct osnoise_context *context) { - long long period_us; - - if (context->period_us != OSNOISE_TIME_INIT_VAL) - return context->period_us; - - if (context->orig_period_us != OSNOISE_TIME_INIT_VAL) - return context->orig_period_us; - - period_us = osnoise_read_ll_config("osnoise/period_us"); - if (period_us < 0) - goto out_err; - - context->orig_period_us = period_us; - return period_us; - -out_err: - return OSNOISE_TIME_INIT_VAL; + return osnoise_read_ll_config("osnoise/period_us", + &context->period_us, + &context->orig_period_us); } static int __osnoise_write_runtime(struct osnoise_context *context, - unsigned long long runtime) + long long runtime) { - int retval; - - if (context->orig_runtime_us == OSNOISE_TIME_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/runtime_us", runtime); - if (retval < 0) - return -1; - - context->runtime_us = runtime; - return 0; + return osnoise_write_ll_config("osnoise/runtime_us", + runtime, + &context->runtime_us, + &context->orig_runtime_us); } static int __osnoise_write_period(struct osnoise_context *context, - unsigned long long period) + long long period) { - int retval; - - if (context->orig_period_us == OSNOISE_TIME_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/period_us", period); - if (retval < 0) - return -1; - - context->period_us = period; - return 0; + return osnoise_write_ll_config("osnoise/period_us", + period, + &context->period_us, + &context->orig_period_us); } /* @@ -258,11 +357,11 @@ static int __osnoise_write_period(struct osnoise_context *context, * to set the runtime and period if they are != 0. */ int osnoise_set_runtime_period(struct osnoise_context *context, - unsigned long long runtime, - unsigned long long period) + long long runtime, + long long period) { - unsigned long long curr_runtime_us; - unsigned long long curr_period_us; + long long curr_runtime_us; + long long curr_period_us; int retval; if (!period && !runtime) @@ -272,7 +371,7 @@ int osnoise_set_runtime_period(struct osnoise_context *context, curr_period_us = osnoise_get_period(context); /* error getting any value? */ - if (curr_period_us == OSNOISE_TIME_INIT_VAL || curr_runtime_us == OSNOISE_TIME_INIT_VAL) + if (curr_period_us == -1 || curr_runtime_us == -1) return -1; if (!period) { @@ -309,13 +408,13 @@ int osnoise_set_runtime_period(struct osnoise_context *context, */ void osnoise_restore_runtime_period(struct osnoise_context *context) { - unsigned long long orig_runtime = context->orig_runtime_us; - unsigned long long orig_period = context->orig_period_us; - unsigned long long curr_runtime = context->runtime_us; - unsigned long long curr_period = context->period_us; + long long orig_runtime = context->orig_runtime_us; + long long orig_period = context->orig_period_us; + long long curr_runtime = context->runtime_us; + long long curr_period = context->period_us; int retval; - if ((orig_runtime == OSNOISE_TIME_INIT_VAL) && (orig_period == OSNOISE_TIME_INIT_VAL)) + if ((orig_runtime == -1) && (orig_period == -1)) return; if ((orig_period == curr_period) && (orig_runtime == curr_runtime)) @@ -326,8 +425,8 @@ void osnoise_restore_runtime_period(struct osnoise_context *context) err_msg("Could not restore original osnoise runtime/period\n"); out_done: - context->runtime_us = OSNOISE_TIME_INIT_VAL; - context->period_us = OSNOISE_TIME_INIT_VAL; + context->runtime_us = -1; + context->period_us = -1; } /* @@ -337,498 +436,81 @@ void osnoise_put_runtime_period(struct osnoise_context *context) { osnoise_restore_runtime_period(context); - if (context->orig_runtime_us != OSNOISE_TIME_INIT_VAL) - context->orig_runtime_us = OSNOISE_TIME_INIT_VAL; + if (context->orig_runtime_us != -1) + context->orig_runtime_us = -1; - if (context->orig_period_us != OSNOISE_TIME_INIT_VAL) - context->orig_period_us = OSNOISE_TIME_INIT_VAL; + if (context->orig_period_us != -1) + context->orig_period_us = -1; } /* - * osnoise_get_timerlat_period_us - read and save the original "timerlat_period_us" + * Long long option set/restore/put functions, generated from OSNOISE_LL_OPTIONS. */ -static long long -osnoise_get_timerlat_period_us(struct osnoise_context *context) -{ - long long timerlat_period_us; - - if (context->timerlat_period_us != OSNOISE_TIME_INIT_VAL) - return context->timerlat_period_us; - - if (context->orig_timerlat_period_us != OSNOISE_TIME_INIT_VAL) - return context->orig_timerlat_period_us; - - timerlat_period_us = osnoise_read_ll_config("osnoise/timerlat_period_us"); - if (timerlat_period_us < 0) - goto out_err; - - context->orig_timerlat_period_us = timerlat_period_us; - return timerlat_period_us; - -out_err: - return OSNOISE_TIME_INIT_VAL; +#define OSNOISE_LL_OPTION(name, path) \ +int osnoise_set_##name(struct osnoise_context *context, long long value) \ +{ \ + return osnoise_write_ll_config(path, \ + value, \ + &context->name, \ + &context->orig_##name); \ +} \ + \ +void osnoise_restore_##name(struct osnoise_context *context) \ +{ \ + osnoise_restore_ll_config(path, &context->name, &context->orig_##name); \ +} \ + \ +static void osnoise_put_##name(struct osnoise_context *context) \ +{ \ + osnoise_restore_##name(context); \ + \ + if (context->orig_##name == -1) \ + return; \ + \ + context->orig_##name = -1; \ } +OSNOISE_LL_OPTIONS +#undef OSNOISE_LL_OPTION -/* - * osnoise_set_timerlat_period_us - set "timerlat_period_us" +/** + * osnoise_get_option - read a boolean flag from osnoise/options + * @option: the option name string to look for (e.g. "OSNOISE_IRQ_DISABLE") + * @set_value: pointer to the cached value set by RTLA, or -1 if unset + * @orig_value: pointer to the cached original value read from tracefs, or -1 if unread + * + * Returns the current state of the flag @option. If RTLA has already written a + * value (@set_value != -1), that value is returned. If the original has already + * been read (@orig_value != -1), that cached value is returned. Otherwise, reads + * the "osnoise/options" file from tracefs, checks whether @option appears with or + * without a "NO_" prefix, caches the result in @orig_value (so it can later be + * restored), and returns it. + * + * This is the shared read primitive used by the OSNOISE_FLAG_OPTION-generated + * set/restore/put functions. + * + * Returns 1 if enabled, 0 if disabled, or -1 on error (option unknown to kernel). */ -int osnoise_set_timerlat_period_us(struct osnoise_context *context, long long timerlat_period_us) +static int osnoise_get_option(char *option, + int *set_value, + int *orig_value) { - long long curr_timerlat_period_us = osnoise_get_timerlat_period_us(context); - int retval; - - if (curr_timerlat_period_us == OSNOISE_TIME_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/timerlat_period_us", timerlat_period_us); - if (retval < 0) - return -1; - - context->timerlat_period_us = timerlat_period_us; - - return 0; -} - -/* - * osnoise_restore_timerlat_period_us - restore "timerlat_period_us" - */ -void osnoise_restore_timerlat_period_us(struct osnoise_context *context) -{ - int retval; - - if (context->orig_timerlat_period_us == OSNOISE_TIME_INIT_VAL) - return; - - if (context->orig_timerlat_period_us == context->timerlat_period_us) - goto out_done; - - retval = osnoise_write_ll_config("osnoise/timerlat_period_us", context->orig_timerlat_period_us); - if (retval < 0) - err_msg("Could not restore original osnoise timerlat_period_us\n"); - -out_done: - context->timerlat_period_us = OSNOISE_TIME_INIT_VAL; -} - -/* - * osnoise_put_timerlat_period_us - restore original values and cleanup data - */ -void osnoise_put_timerlat_period_us(struct osnoise_context *context) -{ - osnoise_restore_timerlat_period_us(context); - - if (context->orig_timerlat_period_us == OSNOISE_TIME_INIT_VAL) - return; - - context->orig_timerlat_period_us = OSNOISE_TIME_INIT_VAL; -} - -/* - * osnoise_get_timerlat_align_us - read and save the original "timerlat_align_us" - */ -static long long -osnoise_get_timerlat_align_us(struct osnoise_context *context) -{ - long long timerlat_align_us; - - if (context->timerlat_align_us != OSNOISE_OPTION_INIT_VAL) - return context->timerlat_align_us; - - if (context->orig_timerlat_align_us != OSNOISE_OPTION_INIT_VAL) - return context->orig_timerlat_align_us; - - timerlat_align_us = osnoise_read_ll_config("osnoise/timerlat_align_us"); - if (timerlat_align_us < 0) - goto out_err; - - context->orig_timerlat_align_us = timerlat_align_us; - return timerlat_align_us; - -out_err: - return OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_set_timerlat_align_us - set "timerlat_align_us" - */ -int osnoise_set_timerlat_align_us(struct osnoise_context *context, long long timerlat_align_us) -{ - long long curr_timerlat_align_us = osnoise_get_timerlat_align_us(context); - int retval; - - if (curr_timerlat_align_us == OSNOISE_OPTION_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/timerlat_align_us", timerlat_align_us); - if (retval < 0) - return -1; - - context->timerlat_align_us = timerlat_align_us; - - return 0; -} - -/* - * osnoise_restore_timerlat_align_us - restore "timerlat_align_us" - */ -void osnoise_restore_timerlat_align_us(struct osnoise_context *context) -{ - int retval; - - if (context->orig_timerlat_align_us == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_timerlat_align_us == context->timerlat_align_us) - goto out_done; - - retval = osnoise_write_ll_config("osnoise/timerlat_align_us", - context->orig_timerlat_align_us); - if (retval < 0) - err_msg("Could not restore original osnoise timerlat_align_us\n"); - -out_done: - context->timerlat_align_us = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_put_timerlat_align_us - restore original values and cleanup data - */ -void osnoise_put_timerlat_align_us(struct osnoise_context *context) -{ - osnoise_restore_timerlat_align_us(context); - - if (context->orig_timerlat_align_us == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_timerlat_align_us = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_get_stop_us - read and save the original "stop_tracing_us" - */ -static long long -osnoise_get_stop_us(struct osnoise_context *context) -{ - long long stop_us; - - if (context->stop_us != OSNOISE_OPTION_INIT_VAL) - return context->stop_us; - - if (context->orig_stop_us != OSNOISE_OPTION_INIT_VAL) - return context->orig_stop_us; - - stop_us = osnoise_read_ll_config("osnoise/stop_tracing_us"); - if (stop_us < 0) - goto out_err; - - context->orig_stop_us = stop_us; - return stop_us; - -out_err: - return OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_set_stop_us - set "stop_tracing_us" - */ -int osnoise_set_stop_us(struct osnoise_context *context, long long stop_us) -{ - long long curr_stop_us = osnoise_get_stop_us(context); - int retval; - - if (curr_stop_us == OSNOISE_OPTION_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/stop_tracing_us", stop_us); - if (retval < 0) - return -1; - - context->stop_us = stop_us; - - return 0; -} - -/* - * osnoise_restore_stop_us - restore the original "stop_tracing_us" - */ -void osnoise_restore_stop_us(struct osnoise_context *context) -{ - int retval; - - if (context->orig_stop_us == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_stop_us == context->stop_us) - goto out_done; - - retval = osnoise_write_ll_config("osnoise/stop_tracing_us", context->orig_stop_us); - if (retval < 0) - err_msg("Could not restore original osnoise stop_us\n"); - -out_done: - context->stop_us = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_put_stop_us - restore original values and cleanup data - */ -void osnoise_put_stop_us(struct osnoise_context *context) -{ - osnoise_restore_stop_us(context); - - if (context->orig_stop_us == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_stop_us = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_get_stop_total_us - read and save the original "stop_tracing_total_us" - */ -static long long -osnoise_get_stop_total_us(struct osnoise_context *context) -{ - long long stop_total_us; - - if (context->stop_total_us != OSNOISE_OPTION_INIT_VAL) - return context->stop_total_us; - - if (context->orig_stop_total_us != OSNOISE_OPTION_INIT_VAL) - return context->orig_stop_total_us; - - stop_total_us = osnoise_read_ll_config("osnoise/stop_tracing_total_us"); - if (stop_total_us < 0) - goto out_err; - - context->orig_stop_total_us = stop_total_us; - return stop_total_us; - -out_err: - return OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_set_stop_total_us - set "stop_tracing_total_us" - */ -int osnoise_set_stop_total_us(struct osnoise_context *context, long long stop_total_us) -{ - long long curr_stop_total_us = osnoise_get_stop_total_us(context); - int retval; - - if (curr_stop_total_us == OSNOISE_OPTION_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/stop_tracing_total_us", stop_total_us); - if (retval < 0) - return -1; - - context->stop_total_us = stop_total_us; - - return 0; -} - -/* - * osnoise_restore_stop_total_us - restore the original "stop_tracing_total_us" - */ -void osnoise_restore_stop_total_us(struct osnoise_context *context) -{ - int retval; - - if (context->orig_stop_total_us == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_stop_total_us == context->stop_total_us) - goto out_done; - - retval = osnoise_write_ll_config("osnoise/stop_tracing_total_us", - context->orig_stop_total_us); - if (retval < 0) - err_msg("Could not restore original osnoise stop_total_us\n"); - -out_done: - context->stop_total_us = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_put_stop_total_us - restore original values and cleanup data - */ -void osnoise_put_stop_total_us(struct osnoise_context *context) -{ - osnoise_restore_stop_total_us(context); - - if (context->orig_stop_total_us == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_stop_total_us = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_get_print_stack - read and save the original "print_stack" - */ -static long long -osnoise_get_print_stack(struct osnoise_context *context) -{ - long long print_stack; - - if (context->print_stack != OSNOISE_OPTION_INIT_VAL) - return context->print_stack; - - if (context->orig_print_stack != OSNOISE_OPTION_INIT_VAL) - return context->orig_print_stack; - - print_stack = osnoise_read_ll_config("osnoise/print_stack"); - if (print_stack < 0) - goto out_err; - - context->orig_print_stack = print_stack; - return print_stack; - -out_err: - return OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_set_print_stack - set "print_stack" - */ -int osnoise_set_print_stack(struct osnoise_context *context, long long print_stack) -{ - long long curr_print_stack = osnoise_get_print_stack(context); - int retval; - - if (curr_print_stack == OSNOISE_OPTION_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("osnoise/print_stack", print_stack); - if (retval < 0) - return -1; - - context->print_stack = print_stack; - - return 0; -} - -/* - * osnoise_restore_print_stack - restore the original "print_stack" - */ -void osnoise_restore_print_stack(struct osnoise_context *context) -{ - int retval; - - if (context->orig_print_stack == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_print_stack == context->print_stack) - goto out_done; - - retval = osnoise_write_ll_config("osnoise/print_stack", context->orig_print_stack); - if (retval < 0) - err_msg("Could not restore original osnoise print_stack\n"); - -out_done: - context->print_stack = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_put_print_stack - restore original values and cleanup data - */ -void osnoise_put_print_stack(struct osnoise_context *context) -{ - osnoise_restore_print_stack(context); - - if (context->orig_print_stack == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_print_stack = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_get_tracing_thresh - read and save the original "tracing_thresh" - */ -static long long -osnoise_get_tracing_thresh(struct osnoise_context *context) -{ - long long tracing_thresh; - - if (context->tracing_thresh != OSNOISE_OPTION_INIT_VAL) - return context->tracing_thresh; - - if (context->orig_tracing_thresh != OSNOISE_OPTION_INIT_VAL) - return context->orig_tracing_thresh; - - tracing_thresh = osnoise_read_ll_config("tracing_thresh"); - if (tracing_thresh < 0) - goto out_err; - - context->orig_tracing_thresh = tracing_thresh; - return tracing_thresh; - -out_err: - return OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_set_tracing_thresh - set "tracing_thresh" - */ -int osnoise_set_tracing_thresh(struct osnoise_context *context, long long tracing_thresh) -{ - long long curr_tracing_thresh = osnoise_get_tracing_thresh(context); - int retval; - - if (curr_tracing_thresh == OSNOISE_OPTION_INIT_VAL) - return -1; - - retval = osnoise_write_ll_config("tracing_thresh", tracing_thresh); - if (retval < 0) - return -1; - - context->tracing_thresh = tracing_thresh; - - return 0; -} - -/* - * osnoise_restore_tracing_thresh - restore the original "tracing_thresh" - */ -void osnoise_restore_tracing_thresh(struct osnoise_context *context) -{ - int retval; - - if (context->orig_tracing_thresh == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_tracing_thresh == context->tracing_thresh) - goto out_done; - - retval = osnoise_write_ll_config("tracing_thresh", context->orig_tracing_thresh); - if (retval < 0) - err_msg("Could not restore original tracing_thresh\n"); - -out_done: - context->tracing_thresh = OSNOISE_OPTION_INIT_VAL; -} - -/* - * osnoise_put_tracing_thresh - restore original values and cleanup data - */ -void osnoise_put_tracing_thresh(struct osnoise_context *context) -{ - osnoise_restore_tracing_thresh(context); - - if (context->orig_tracing_thresh == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_tracing_thresh = OSNOISE_OPTION_INIT_VAL; -} - -static int osnoise_options_get_option(char *option) -{ - char *options = tracefs_instance_file_read(NULL, "osnoise/options", NULL); + char *options; char no_option[128]; int retval = 0; char *opt; + if (*set_value != -1) + /* option has been set by RTLA already */ + return *set_value; + + if (*orig_value != -1) + /* RTLA has already read the option */ + return *orig_value; + + /* current value is not known to RTLA yet, read it from tracefs */ + options = tracefs_instance_file_read(NULL, "osnoise/options", NULL); if (!options) - return OSNOISE_OPTION_INIT_VAL; + return -1; /* * Check first if the option is disabled. @@ -847,207 +529,138 @@ static int osnoise_options_get_option(char *option) if (opt) retval = 1; else - retval = OSNOISE_OPTION_INIT_VAL; + retval = -1; out_free: free(options); + + if (retval < 0) + goto out_err; + + /* save the value and return it */ + *orig_value = retval; return retval; + +out_err: + return -1; } -static int osnoise_options_set_option(char *option, bool onoff) +/** + * osnoise_set_option - write a boolean flag to osnoise/options + * @option: the option name string (e.g. "OSNOISE_IRQ_DISABLE") + * @onoff: the desired state (true to enable, false to disable) + * @set_value: pointer to the cached value set by RTLA, or -1 if unset + * @orig_value: pointer to the cached original value read from tracefs, or -1 if unread + * + * Sets the flag @option to the state @onoff. Before writing, calls + * osnoise_get_option() to ensure the original value is cached in @orig_value + * (enabling later restoration). If the current value already matches @onoff, + * no write is performed. On successful write, records the new state in + * @set_value. + * + * This is the shared write primitive used by the OSNOISE_FLAG_OPTION-generated + * set/restore/put functions. + * + * Returns 0 on success, -1 on read error (option likely unknown to kernel), + * or -2 on write error. + */ +static int osnoise_set_option(char *option, + bool onoff, + int *set_value, + int *orig_value) { + int curr = osnoise_get_option(option, set_value, orig_value); char no_option[128]; - - if (onoff) - return tracefs_instance_file_write(NULL, "osnoise/options", option); - - snprintf(no_option, sizeof(no_option), "NO_%s", option); - - return tracefs_instance_file_write(NULL, "osnoise/options", no_option); -} - -static int osnoise_get_irq_disable(struct osnoise_context *context) -{ - if (context->opt_irq_disable != OSNOISE_OPTION_INIT_VAL) - return context->opt_irq_disable; - - if (context->orig_opt_irq_disable != OSNOISE_OPTION_INIT_VAL) - return context->orig_opt_irq_disable; - - context->orig_opt_irq_disable = osnoise_options_get_option("OSNOISE_IRQ_DISABLE"); - - return context->orig_opt_irq_disable; -} - -int osnoise_set_irq_disable(struct osnoise_context *context, bool onoff) -{ - int opt_irq_disable = osnoise_get_irq_disable(context); int retval; - if (opt_irq_disable == OSNOISE_OPTION_INIT_VAL) + if (curr == -1) + /* read failed, option likely unknown to kernel */ return -1; - if (opt_irq_disable == onoff) + if (curr == onoff) return 0; - retval = osnoise_options_set_option("OSNOISE_IRQ_DISABLE", onoff); + if (onoff) { + retval = tracefs_instance_file_write(NULL, "osnoise/options", option); + } else { + snprintf(no_option, sizeof(no_option), "NO_%s", option); + retval = tracefs_instance_file_write(NULL, "osnoise/options", no_option); + } + if (retval < 0) - return -1; - - context->opt_irq_disable = onoff; - - return 0; -} - -static void osnoise_restore_irq_disable(struct osnoise_context *context) -{ - int retval; - - if (context->orig_opt_irq_disable == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_opt_irq_disable == context->opt_irq_disable) - goto out_done; - - retval = osnoise_options_set_option("OSNOISE_IRQ_DISABLE", context->orig_opt_irq_disable); - if (retval < 0) - err_msg("Could not restore original OSNOISE_IRQ_DISABLE option\n"); - -out_done: - context->orig_opt_irq_disable = OSNOISE_OPTION_INIT_VAL; -} - -static void osnoise_put_irq_disable(struct osnoise_context *context) -{ - osnoise_restore_irq_disable(context); - - if (context->orig_opt_irq_disable == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_opt_irq_disable = OSNOISE_OPTION_INIT_VAL; -} - -static int osnoise_get_workload(struct osnoise_context *context) -{ - if (context->opt_workload != OSNOISE_OPTION_INIT_VAL) - return context->opt_workload; - - if (context->orig_opt_workload != OSNOISE_OPTION_INIT_VAL) - return context->orig_opt_workload; - - context->orig_opt_workload = osnoise_options_get_option("OSNOISE_WORKLOAD"); - - return context->orig_opt_workload; -} - -int osnoise_set_workload(struct osnoise_context *context, bool onoff) -{ - int opt_workload = osnoise_get_workload(context); - int retval; - - if (opt_workload == OSNOISE_OPTION_INIT_VAL) - return -1; - - if (opt_workload == onoff) - return 0; - - retval = osnoise_options_set_option("OSNOISE_WORKLOAD", onoff); - if (retval < 0) + /* write failed, hard error */ return -2; - context->opt_workload = onoff; - + /* record the set value and return success */ + *set_value = onoff; return 0; } -static void osnoise_restore_workload(struct osnoise_context *context) +/** + * osnoise_restore_option - restore a boolean flag to its original value + * @option: the option name string (e.g. "OSNOISE_IRQ_DISABLE") + * @set_value: pointer to the cached value set by RTLA, or -1 if unset + * @orig_value: pointer to the cached original value read from tracefs, or -1 if unread + * + * Restores the flag @option to the state cached in @orig_value (which was saved + * by a prior osnoise_get_option() or osnoise_set_option() call). If the original + * was never read, or if the current set value already matches the original, no + * write is performed. After restoring, clears @set_value to -1 to indicate RTLA + * no longer overrides this option. + * + * This is the shared restore primitive used by the OSNOISE_FLAG_OPTION-generated + * restore/put functions. + */ +static void osnoise_restore_option(char *option, + int *set_value, + int *orig_value) { int retval; - if (context->orig_opt_workload == OSNOISE_OPTION_INIT_VAL) + if (*orig_value == -1) return; - if (context->orig_opt_workload == context->opt_workload) + if (*orig_value == *set_value) goto out_done; - retval = osnoise_options_set_option("OSNOISE_WORKLOAD", context->orig_opt_workload); + retval = osnoise_set_option(option, *orig_value, set_value, orig_value); if (retval < 0) - err_msg("Could not restore original OSNOISE_WORKLOAD option\n"); + err_msg("Could not restore original %s option\n", option); out_done: - context->orig_opt_workload = OSNOISE_OPTION_INIT_VAL; + *set_value = -1; } -static void osnoise_put_workload(struct osnoise_context *context) -{ - osnoise_restore_workload(context); - - if (context->orig_opt_workload == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_opt_workload = OSNOISE_OPTION_INIT_VAL; +/* + * Flag option set/restore/put functions, generated from OSNOISE_FLAG_OPTIONS. + */ +#define OSNOISE_FLAG_OPTION(name, option_str) \ +int osnoise_set_##name(struct osnoise_context *context, bool onoff) \ +{ \ + return osnoise_set_option(option_str, \ + onoff, \ + &context->opt_##name, \ + &context->orig_opt_##name); \ +} \ + \ +void osnoise_restore_##name(struct osnoise_context *context) \ +{ \ + osnoise_restore_option(option_str, \ + &context->opt_##name, \ + &context->orig_opt_##name); \ +} \ + \ +static void osnoise_put_##name(struct osnoise_context *context) \ +{ \ + osnoise_restore_##name(context); \ + \ + if (context->orig_opt_##name == -1) \ + return; \ + \ + context->orig_opt_##name = -1; \ } - -static int osnoise_get_timerlat_align(struct osnoise_context *context) -{ - if (context->opt_timerlat_align != OSNOISE_OPTION_INIT_VAL) - return context->opt_timerlat_align; - - if (context->orig_opt_timerlat_align != OSNOISE_OPTION_INIT_VAL) - return context->orig_opt_timerlat_align; - - context->orig_opt_timerlat_align = osnoise_options_get_option("TIMERLAT_ALIGN"); - - return context->orig_opt_timerlat_align; -} - -int osnoise_set_timerlat_align(struct osnoise_context *context, bool onoff) -{ - int opt_timerlat_align = osnoise_get_timerlat_align(context); - int retval; - - if (opt_timerlat_align == OSNOISE_OPTION_INIT_VAL) - return -1; - - if (opt_timerlat_align == onoff) - return 0; - - retval = osnoise_options_set_option("TIMERLAT_ALIGN", onoff); - if (retval < 0) - return -2; - - context->opt_timerlat_align = onoff; - - return 0; -} - -static void osnoise_restore_timerlat_align(struct osnoise_context *context) -{ - int retval; - - if (context->orig_opt_timerlat_align == OSNOISE_OPTION_INIT_VAL) - return; - - if (context->orig_opt_timerlat_align == context->opt_timerlat_align) - goto out_done; - - retval = osnoise_options_set_option("TIMERLAT_ALIGN", context->orig_opt_timerlat_align); - if (retval < 0) - err_msg("Could not restore original TIMERLAT_ALIGN option\n"); - -out_done: - context->orig_opt_timerlat_align = OSNOISE_OPTION_INIT_VAL; -} - -static void osnoise_put_timerlat_align(struct osnoise_context *context) -{ - osnoise_restore_timerlat_align(context); - - if (context->orig_opt_timerlat_align == OSNOISE_OPTION_INIT_VAL) - return; - - context->orig_opt_timerlat_align = OSNOISE_OPTION_INIT_VAL; -} +OSNOISE_FLAG_OPTIONS +#undef OSNOISE_FLAG_OPTION enum { FLAG_CONTEXT_NEWLY_CREATED = (1 << 0), @@ -1083,29 +696,23 @@ struct osnoise_context *osnoise_context_alloc(void) context = calloc_fatal(1, sizeof(*context)); - context->orig_stop_us = OSNOISE_OPTION_INIT_VAL; - context->stop_us = OSNOISE_OPTION_INIT_VAL; + /* First allocate manually implemented options... */ + context->orig_runtime_us = -1; + context->runtime_us = -1; + context->orig_period_us = -1; + context->period_us = -1; - context->orig_stop_total_us = OSNOISE_OPTION_INIT_VAL; - context->stop_total_us = OSNOISE_OPTION_INIT_VAL; - - context->orig_print_stack = OSNOISE_OPTION_INIT_VAL; - context->print_stack = OSNOISE_OPTION_INIT_VAL; - - context->orig_tracing_thresh = OSNOISE_OPTION_INIT_VAL; - context->tracing_thresh = OSNOISE_OPTION_INIT_VAL; - - context->orig_opt_irq_disable = OSNOISE_OPTION_INIT_VAL; - context->opt_irq_disable = OSNOISE_OPTION_INIT_VAL; - - context->orig_opt_workload = OSNOISE_OPTION_INIT_VAL; - context->opt_workload = OSNOISE_OPTION_INIT_VAL; - - context->orig_opt_timerlat_align = OSNOISE_OPTION_INIT_VAL; - context->opt_timerlat_align = OSNOISE_OPTION_INIT_VAL; - - context->orig_timerlat_align_us = OSNOISE_OPTION_INIT_VAL; - context->timerlat_align_us = OSNOISE_OPTION_INIT_VAL; + /* ...then the automatically generated ones. */ +#define OSNOISE_LL_OPTION(name, path) \ + context->orig_##name = -1; \ + context->name = -1; +#define OSNOISE_FLAG_OPTION(name, option_str) \ + context->orig_opt_##name = -1; \ + context->opt_##name = -1; + OSNOISE_LL_OPTIONS + OSNOISE_FLAG_OPTIONS +#undef OSNOISE_LL_OPTION +#undef OSNOISE_FLAG_OPTION osnoise_get_context(context); @@ -1126,17 +733,17 @@ void osnoise_put_context(struct osnoise_context *context) if (!(context->flags & FLAG_CONTEXT_DELETED)) return; + /* First restore the original values of the options... */ osnoise_put_cpus(context); osnoise_put_runtime_period(context); - osnoise_put_stop_us(context); - osnoise_put_stop_total_us(context); - osnoise_put_timerlat_period_us(context); - osnoise_put_print_stack(context); - osnoise_put_tracing_thresh(context); - osnoise_put_irq_disable(context); - osnoise_put_workload(context); - osnoise_put_timerlat_align(context); - osnoise_put_timerlat_align_us(context); + + /* ...then the automatically generated ones. */ +#define OSNOISE_LL_OPTION(name, path) osnoise_put_##name(context); +#define OSNOISE_FLAG_OPTION(name, option_str) osnoise_put_##name(context); + OSNOISE_LL_OPTIONS + OSNOISE_FLAG_OPTIONS +#undef OSNOISE_LL_OPTION +#undef OSNOISE_FLAG_OPTION free(context); } @@ -1179,9 +786,55 @@ struct osnoise_tool *osnoise_init_tool(char *tool_name) } /* + * osnoise_init_ipi_filters - Initialize event filtering for IPI events + */ +int osnoise_init_ipi_filters(struct osnoise_tool *tool, + struct common_params *params, + bool *filters_enabled) +{ + char filter[MAX_PATH]; + int retval; + /* + * If tracing on a subset of possible CPUs, leverage the kernel filtering + * infrastructure to only generate events on traced CPUs. + * Older kernels (pre v6.6) may have the IPI events but not the ability + * to filter them, so allow that to fail gracefully. + */ + + snprintf(filter, ARRAY_SIZE(filter), "cpu & CPUS{%s}\n", params->cpus); + retval = tracefs_event_file_write(tool->trace.inst, + "ipi", "ipi_send_cpu", "filter", + filter); + if (retval < 0) { + debug_msg("Could not set ipi_send_cpu CPU filter\n"); + *filters_enabled = false; + return 0; + } + + + snprintf(filter, ARRAY_SIZE(filter), "cpumask & CPUS{%s}\n", params->cpus); + retval = tracefs_event_file_write(tool->trace.inst, + "ipi", "ipi_send_cpumask", "filter", + filter); + if (retval < 0) { + /* + * If we managed to set up the previous filter but not + * this one, something's really wrong + */ + err_msg("Could not set ipi_send_cpumask CPU filter\n"); + *filters_enabled = false; + return -1; + } + + *filters_enabled = true; + return 0; +} + +/* * osnoise_init_trace_tool - init a tracer instance to trace osnoise events */ -struct osnoise_tool *osnoise_init_trace_tool(const char *tracer) +struct osnoise_tool *osnoise_init_trace_tool(struct common_params *params, + const char *tracer) { struct osnoise_tool *trace; int retval; @@ -1196,6 +849,30 @@ struct osnoise_tool *osnoise_init_trace_tool(const char *tracer) goto out_err; } + if (!params->ipi) + goto done; + + retval = tracefs_event_enable(trace->trace.inst, "ipi", "ipi_send_cpu"); + if (retval < 0 && !errno) { + err_msg("Could not find ipi_send_cpu event\n"); + goto out_err; + } + + retval = tracefs_event_enable(trace->trace.inst, "ipi", "ipi_send_cpumask"); + if (retval < 0 && !errno) { + err_msg("Could not find ipi_send_cpumask event\n"); + goto out_err; + } + + if (params->cpus) { + bool unused; + + retval = osnoise_init_ipi_filters(trace, params, &unused); + if (retval < 0) + goto out_err; + } + +done: retval = enable_tracer_by_name(trace->trace.inst, tracer); if (retval) { err_msg("Could not enable %s tracer for tracing\n", tracer); @@ -1225,6 +902,29 @@ bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record } /* + * osnoise_report_invalid_events - report number of invalid events + */ +void +osnoise_report_invalid_events(struct osnoise_tool *tool) +{ + unsigned long long total_events; + + if (tool->trace.invalid_events > 0) { + if (tool->trace.missed_events != UINT64_MAX) { + total_events = tool->trace.processed_events + tool->trace.invalid_events + + tool->trace.missed_events; + + printf("%lld (%.2f%%) invalid events, results might not be accurate\n", + tool->trace.invalid_events, + (double) tool->trace.invalid_events / total_events * 100.0); + } else { + printf("%lld invalid events, results might not be accurate\n", + tool->trace.invalid_events); + } + } +} + +/* * osnoise_report_missed_events - report number of events dropped by trace * buffer */ @@ -1236,7 +936,8 @@ osnoise_report_missed_events(struct osnoise_tool *tool) if (tool->trace.missed_events == UINT64_MAX) printf("unknown number of events missed, results might not be accurate\n"); else if (tool->trace.missed_events > 0) { - total_events = tool->trace.processed_events + tool->trace.missed_events; + total_events = tool->trace.processed_events + tool->trace.invalid_events + + tool->trace.missed_events; printf("%lld (%.2f%%) events missed, results might not be accurate\n", tool->trace.missed_events,
diff --git a/tools/tracing/rtla/src/osnoise.h b/tools/tracing/rtla/src/osnoise.h index 340ff5a..32f0555 100644 --- a/tools/tracing/rtla/src/osnoise.h +++ b/tools/tracing/rtla/src/osnoise.h
@@ -18,44 +18,16 @@ struct osnoise_params { #define to_osnoise_params(ptr) container_of(ptr, struct osnoise_params, common) -/* - * *_INIT_VALs are also invalid values, they are used to - * communicate errors. - */ -#define OSNOISE_OPTION_INIT_VAL (-1) -#define OSNOISE_TIME_INIT_VAL (0) - struct osnoise_context *osnoise_context_alloc(void); int osnoise_get_context(struct osnoise_context *context); void osnoise_put_context(struct osnoise_context *context); int osnoise_set_runtime_period(struct osnoise_context *context, - unsigned long long runtime, - unsigned long long period); + long long runtime, + long long period); void osnoise_restore_runtime_period(struct osnoise_context *context); -void osnoise_restore_stop_us(struct osnoise_context *context); -void osnoise_restore_stop_total_us(struct osnoise_context *context); - -int osnoise_set_timerlat_period_us(struct osnoise_context *context, - long long timerlat_period_us); -void osnoise_restore_timerlat_period_us(struct osnoise_context *context); - -int osnoise_set_tracing_thresh(struct osnoise_context *context, - long long tracing_thresh); -void osnoise_restore_tracing_thresh(struct osnoise_context *context); - -void osnoise_restore_print_stack(struct osnoise_context *context); -int osnoise_set_print_stack(struct osnoise_context *context, - long long print_stack); - -int osnoise_set_timerlat_align_us(struct osnoise_context *context, - long long timerlat_align_us); -void osnoise_restore_timerlat_align_us(struct osnoise_context *context); - -int osnoise_set_timerlat_align(struct osnoise_context *context, bool onoff); - -int osnoise_set_irq_disable(struct osnoise_context *context, bool onoff); +void osnoise_report_invalid_events(struct osnoise_tool *tool); void osnoise_report_missed_events(struct osnoise_tool *tool); int osnoise_apply_config(struct osnoise_tool *tool, struct osnoise_params *params); @@ -63,6 +35,10 @@ int osnoise_enable(struct osnoise_tool *tool); int osnoise_main(int argc, char **argv); int hwnoise_main(int argc, char **argv); +int osnoise_init_ipi_filters(struct osnoise_tool *tool, + struct common_params *params, + bool *filters_enabled); + extern struct tool_ops timerlat_top_ops, timerlat_hist_ops; extern struct tool_ops osnoise_top_ops, osnoise_hist_ops;
diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index dfa91d0..bad0b89 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c
@@ -397,6 +397,7 @@ osnoise_print_stats(struct osnoise_tool *tool) trace_seq_reset(trace->seq); osnoise_print_summary(params, trace, data); + osnoise_report_invalid_events(tool); osnoise_report_missed_events(tool); }
diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 512a629..1169076 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c
@@ -8,6 +8,7 @@ #include <string.h> #include <signal.h> #include <unistd.h> +#include <errno.h> #include <stdio.h> #include <time.h> @@ -25,6 +26,7 @@ struct osnoise_top_cpu { unsigned long long irq_count; unsigned long long softirq_count; unsigned long long thread_count; + unsigned long long ipi_count; int sum_cycles; }; @@ -164,6 +166,8 @@ static void osnoise_top_header(struct osnoise_tool *top) goto eol; trace_seq_printf(s, " IRQ Softirq Thread"); + if (params->common.ipi) + trace_seq_printf(s, " IPI"); eol: if (pretty) @@ -218,7 +222,10 @@ static void osnoise_top_print(struct osnoise_tool *tool, int cpu) trace_seq_printf(s, "%12llu ", cpu_data->irq_count); trace_seq_printf(s, "%12llu ", cpu_data->softirq_count); - trace_seq_printf(s, "%12llu\n", cpu_data->thread_count); + trace_seq_printf(s, "%12llu", cpu_data->thread_count); + if (params->common.ipi) + trace_seq_printf(s, " %12llu", cpu_data->ipi_count); + trace_seq_printf(s, "\n"); } /* @@ -242,6 +249,7 @@ osnoise_print_stats(struct osnoise_tool *top) trace_seq_do_printf(trace->seq); trace_seq_reset(trace->seq); + osnoise_report_invalid_events(top); osnoise_report_missed_events(top); } @@ -275,12 +283,119 @@ osnoise_top_apply_config(struct osnoise_tool *tool) return -1; } +static void account_ipi(struct osnoise_tool *tool, unsigned long long dst_cpu) +{ + struct osnoise_top_cpu *cpu_data; + struct osnoise_top_data *data; + unsigned long long inc = 1; + + data = tool->data; + cpu_data = &data->cpu_data[dst_cpu]; + + update_sum(&cpu_data->ipi_count, &inc); +} + +/* + * osnoise_ipi_cpu_handler - this is the handler for single CPU IPI events. + */ +static int +osnoise_ipi_cpu_handler(struct trace_seq *s, struct tep_record *record, + struct tep_event *event, void *context) +{ + struct osnoise_tool *tool; + unsigned long long dst_cpu; + struct trace_instance *trace = context; + + tool = container_of(trace, struct osnoise_tool, trace); + tep_get_field_val(s, event, "cpu", record, &dst_cpu, 1); + + account_ipi(tool, dst_cpu); + + return 0; +} + +/* + * osnoise_ipi_cpu_unfiltered_handler - this is the handler for single CPU IPI + * events. Slightly less optimized than + * the filtered variant. + */ +static int +osnoise_ipi_cpu_unfiltered_handler(struct trace_seq *s, struct tep_record *record, + struct tep_event *event, void *context) +{ + struct osnoise_tool *tool; + unsigned long long dst_cpu; + struct osnoise_params *params; + struct trace_instance *trace = context; + + tool = container_of(trace, struct osnoise_tool, trace); + params = to_osnoise_params(tool->params); + tep_get_field_val(s, event, "cpu", record, &dst_cpu, 1); + + if (CPU_ISSET(dst_cpu, ¶ms->common.monitored_cpus)) + account_ipi(tool, dst_cpu); + + return 0; +} + +static cpu_set_t cpumask_tmp_cpus; + +/* + * osnoise_ipi_cpumask_handler - this is the handler for broadcasted IPI events. + */ +static int +osnoise_ipi_cpumask_handler(struct trace_seq *s, struct tep_record *record, + struct tep_event *event, void *context) +{ + struct trace_instance *trace = context; + struct osnoise_tool *tool; + struct osnoise_params *params; + struct tep_format_field *field; + cpu_set_t *event_cpus; + int len; + + tool = container_of(trace, struct osnoise_tool, trace); + params = to_osnoise_params(tool->params); + + field = tep_find_field(event, "cpumask"); + if (!field) + return 0; + + event_cpus = tep_get_field_raw(s, event, "cpumask", record, &len, 1); + if (!event_cpus) { + err_msg("Failed to get cpumask field\n"); + return 0; + } + + /* + * Even if already filtering for such an intersection, we need to compute + * the intersection here as the @cpumask field may contain non-monitored + * CPUs. + */ + CPU_AND(&cpumask_tmp_cpus, event_cpus, ¶ms->common.monitored_cpus); + + /* + * Computing the mask weight is overkill but there is no leaner option + * provided by glibc, e.g cpumask_first() or somesuch. + */ + if (CPU_COUNT(&cpumask_tmp_cpus)) { + for (int cpu = 0; cpu < nr_cpus; cpu++) { + if (CPU_ISSET(cpu, &cpumask_tmp_cpus)) + account_ipi(tool, cpu); + } + } + + return 0; +} + /* * osnoise_init_top - initialize a osnoise top tool with parameters */ struct osnoise_tool *osnoise_init_top(struct common_params *params) { + bool ipi_filters_enabled; struct osnoise_tool *tool; + int retval; tool = osnoise_init_tool("osnoise_top"); if (!tool) @@ -295,7 +410,46 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params) tep_register_event_handler(tool->trace.tep, -1, "ftrace", "osnoise", osnoise_top_handler, NULL); + if (!params->ipi) + goto out; + + retval = tracefs_event_enable(tool->trace.inst, "ipi", "ipi_send_cpu"); + if (retval < 0 && !errno) { + err_msg("Could not find ipi_send_cpu event\n"); + goto out_err; + } + + retval = tracefs_event_enable(tool->trace.inst, "ipi", "ipi_send_cpumask"); + if (retval < 0 && !errno) { + err_msg("Could not find ipi_send_cpumask event\n"); + goto out_err; + } + + if (params->cpus) { + retval = osnoise_init_ipi_filters(tool, params, &ipi_filters_enabled); + if (retval < 0) + goto out_err; + } + + /* + * If no filtering is available and we're tracing all CPUs, we can still + * use the filtered callback since stats are collected for all CPUs. + */ + tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpu", + (!params->cpus || ipi_filters_enabled) ? + osnoise_ipi_cpu_handler : + osnoise_ipi_cpu_unfiltered_handler, + NULL); + + tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpumask", + osnoise_ipi_cpumask_handler, NULL); + +out: return tool; +out_err: + osnoise_free_top_tool(tool); + osnoise_destroy_tool(tool); + return NULL; } struct tool_ops osnoise_top_ops = {
diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index df7b139..b6af5ba 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c
@@ -682,6 +682,7 @@ timerlat_print_stats(struct osnoise_tool *tool) timerlat_print_summary(params, trace, data); timerlat_print_stats_all(params, trace, data); + osnoise_report_invalid_events(tool); osnoise_report_missed_events(tool); }
diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 6206a0a..2afd619 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c
@@ -456,6 +456,7 @@ timerlat_print_stats(struct osnoise_tool *top) trace_seq_do_printf(trace->seq); trace_seq_reset(trace->seq); + osnoise_report_invalid_events(top); osnoise_report_missed_events(top); }
diff --git a/tools/tracing/rtla/src/trace.c b/tools/tracing/rtla/src/trace.c index e407447..4b5f4d0 100644 --- a/tools/tracing/rtla/src/trace.c +++ b/tools/tracing/rtla/src/trace.c
@@ -138,6 +138,12 @@ collect_registered_events(struct tep_event *event, struct tep_record *record, struct trace_instance *trace = context; struct trace_seq *s = trace->seq; + if (cpu >= nr_cpus) { + /* Kernel reports event on CPU we don't see, corrupt data? */ + trace->invalid_events++; + return 0; + } + trace->processed_events++; if (!event->handler) @@ -236,6 +242,7 @@ int trace_instance_init(struct trace_instance *trace, char *tool_name) trace); trace->processed_events = 0; + trace->invalid_events = 0; return 0; @@ -440,6 +447,21 @@ static void trace_event_disable_trigger(struct trace_instance *instance, tevent->event ? : "*", tevent->trigger); } +static inline struct trace_events *trace_events_tail(struct trace_events *tevent) +{ + while (tevent && tevent->next) + tevent = tevent->next; + + return tevent; +} + +/* + * Events are stashed in LIFO order; flip that to FIFO to process them in the + * same order as they are defined by the user on the command line. + */ +#define for_each_trace_event(tevent) \ + for (tevent = trace_events_tail(tevent); tevent; tevent = tevent->prev) + /* * trace_events_disable - disable all trace events */ @@ -451,7 +473,7 @@ void trace_events_disable(struct trace_instance *instance, if (!events) return; - while (tevent) { + for_each_trace_event(tevent) { debug_msg("Disabling event %s:%s\n", tevent->system, tevent->event ? : "*"); if (tevent->enabled) { trace_event_disable_filter(instance, tevent); @@ -460,7 +482,6 @@ void trace_events_disable(struct trace_instance *instance, } tevent->enabled = 0; - tevent = tevent->next; } } @@ -473,6 +494,10 @@ static int trace_event_enable_filter(struct trace_instance *instance, char filter[MAX_PATH]; int retval; + /* Unconditionally clean any pre-existing filters */ + tracefs_event_file_write(instance->inst, tevent->system, + tevent->event, "filter", "0"); + if (!tevent->filter) return 0; @@ -544,7 +569,10 @@ int trace_events_enable(struct trace_instance *instance, struct trace_events *tevent = events; int retval; - while (tevent) { + if (!events) + return 0; + + for_each_trace_event(tevent) { debug_msg("Enabling event %s:%s\n", tevent->system, tevent->event ? : "*"); retval = tracefs_event_enable(instance->inst, tevent->system, tevent->event); if (retval < 0) { @@ -562,7 +590,6 @@ int trace_events_enable(struct trace_instance *instance, return 1; tevent->enabled = 1; - tevent = tevent->next; } return 0;
diff --git a/tools/tracing/rtla/src/trace.h b/tools/tracing/rtla/src/trace.h index 95b911a..94e4510 100644 --- a/tools/tracing/rtla/src/trace.h +++ b/tools/tracing/rtla/src/trace.h
@@ -4,6 +4,7 @@ struct trace_events { struct trace_events *next; + struct trace_events *prev; char *system; char *event; char *filter; @@ -18,6 +19,7 @@ struct trace_instance { struct tep_handle *tep; struct trace_seq *seq; unsigned long long missed_events; + unsigned long long invalid_events; unsigned long long processed_events; };
diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index cb187e7..5d91c91 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c
@@ -108,6 +108,169 @@ void get_duration(time_t start_time, char *output, int output_size) } /* + * cpu_list_iterate - parse a cpu list and call a function on each element + * + * If callback returns a non-zero value, the iteration is stopped. + * + * Returns the number of cpus in the list (including duplicates) on success, + * callback return value on break, and -1 on error. + */ +int cpu_list_iterate(const char *cpu_list, int (*callback)(int, void *), void *data) +{ + const char *p; + int i, cpu, end_cpu, count = 0, retval; + + for (p = cpu_list; *p && *p != '\n'; ) { + cpu = atoi(p); + if (cpu < 0 || (!cpu && *p != '0')) + return -1; + + while (isdigit(*p)) + p++; + if (*p == '-') { + p++; + end_cpu = atoi(p); + if (end_cpu < cpu || (!end_cpu && *p != '0')) + return -1; + while (isdigit(*p)) + p++; + } else + end_cpu = cpu; + + for (i = cpu; i <= end_cpu; i++) { + retval = callback(i, data); + if (retval) + return retval; + ++count; + } + + if (*p == ',') + p++; + } + + return count; +} + +static int max_cpu_callback(int i, void *data) +{ + int *max_cpu = data; + + if (i > *max_cpu) + *max_cpu = i; + + return 0; +} + +/* + * get_max_cpu_from_list - get the maximum CPU in a CPU list + * + * Returns maximum CPU number, or -1 on error + */ +int get_max_cpu_from_list(const char *cpu_list) +{ + int max_cpu = -1; + + if (cpu_list_iterate(cpu_list, max_cpu_callback, &max_cpu) < 0) + return -1; + + return max_cpu; +} + + +static int tmp_cpu_set_callback(int i, void *data) +{ + bool *cpu_set = data; + + cpu_set[i] = true; + + return 0; +} + +/* + * get_possible_cpus - get the number of possible CPUs from sysfs + * + * Parse /sys/devices/system/cpu/possible to determine the number of + * possible CPUs. Only contiguous zero-based CPUs lists are accepted. + * + * Returns the number of possible CPUs, or a negative value on error: + * - -1 if the file is unreadable, + * - -2 if parsing failed, + * - -3 if the cpu list is non-zero-based or non-contiguous. + */ +int get_possible_cpus(void) +{ + char *str = NULL; + size_t len = 0; + int nr_cpus = 0, max_cpu = -1, i; + FILE *fp; + bool *cpu_set; + + fp = fopen("/sys/devices/system/cpu/possible", "r"); + if (!fp) + return -1; + + if (getline(&str, &len, fp) < 1) { + /* cpu string should be at least 1 character */ + if (str) + free(str); + fclose(fp); + return -1; + } + + fclose(fp); + + /* get maximum cpu number */ + if (cpu_list_iterate(str, max_cpu_callback, &max_cpu) < 0) { + free(str); + return -2; + } + + if (max_cpu < 0 || max_cpu == INT_MAX) { + /* empty or bogus cpu list */ + free(str); + return -2; + } + + /* get max cpu using dynamic array, as nr_cpus might be > 1024 */ + cpu_set = calloc(max_cpu + 1, sizeof(bool)); + if (!cpu_set) { + free(str); + return -2; + } + if (cpu_list_iterate(str, tmp_cpu_set_callback, cpu_set) < 0) { + free(str); + free(cpu_set); + return -2; + } + for (i = 0; i <= max_cpu; i++) { + if (cpu_set[i]) + ++nr_cpus; + } + free(cpu_set); + + free(str); + + if (max_cpu >= nr_cpus) + /* rtla assumes cpu < nr_cpus for all cpus */ + return -3; + + return nr_cpus; +} + +static int cpu_set_callback(int i, void *data) +{ + cpu_set_t *set = data; + + if (i >= nr_cpus || i >= CPU_SETSIZE) + return -1; + + debug_msg("cpu_set: adding cpu %d\n", i); + CPU_SET(i, set); + + return 0; +} + +/* * parse_cpu_set - parse a cpu_list filling cpu_set_t argument * * Receives a cpu list, like 1-3,5 (cpus 1, 2, 3, 5), and then set @@ -117,48 +280,14 @@ void get_duration(time_t start_time, char *output, int output_size) */ int parse_cpu_set(char *cpu_list, cpu_set_t *set) { - const char *p; - int end_cpu; - int cpu; - int i; - CPU_ZERO(set); - for (p = cpu_list; *p; ) { - cpu = atoi(p); - if (cpu < 0 || (!cpu && *p != '0') || cpu >= nr_cpus) - goto err; - - while (isdigit(*p)) - p++; - if (*p == '-') { - p++; - end_cpu = atoi(p); - if (end_cpu < cpu || (!end_cpu && *p != '0') || end_cpu >= nr_cpus) - goto err; - while (isdigit(*p)) - p++; - } else - end_cpu = cpu; - - if (cpu == end_cpu) { - debug_msg("cpu_set: adding cpu %d\n", cpu); - CPU_SET(cpu, set); - } else { - for (i = cpu; i <= end_cpu; i++) { - debug_msg("cpu_set: adding cpu %d\n", i); - CPU_SET(i, set); - } - } - - if (*p == ',') - p++; + if (cpu_list_iterate(cpu_list, cpu_set_callback, set) < 0) { + debug_msg("Error parsing the cpu set %s\n", cpu_list); + return 1; } return 0; -err: - debug_msg("Error parsing the cpu set %s\n", cpu_list); - return 1; } /*
diff --git a/tools/tracing/rtla/src/utils.h b/tools/tracing/rtla/src/utils.h index 2ba3333..44b4506 100644 --- a/tools/tracing/rtla/src/utils.h +++ b/tools/tracing/rtla/src/utils.h
@@ -40,6 +40,8 @@ static inline bool str_has_prefix(const char *str, const char *prefix) } extern bool config_debug; +extern int nr_cpus; + void debug_msg(const char *fmt, ...); void err_msg(const char *fmt, ...); void fatal(const char *fmt, ...); @@ -47,6 +49,10 @@ void fatal(const char *fmt, ...); long parse_seconds_duration(char *val); void get_duration(time_t start_time, char *output, int output_size); +int cpu_list_iterate(const char *cpu_list, int (*callback)(int, void *), void *data); +int get_max_cpu_from_list(const char *cpu_list); +int get_possible_cpus(void); + long long get_llong_from_str(char *start); static inline void
diff --git a/tools/tracing/rtla/tests/engine.sh b/tools/tracing/rtla/tests/engine.sh index 4287cd64..395b472 100644 --- a/tools/tracing/rtla/tests/engine.sh +++ b/tools/tracing/rtla/tests/engine.sh
@@ -7,8 +7,14 @@ # Set test directory to the directory of the script scriptfile=$(realpath "$0") testdir=$(dirname "$scriptfile") - [ -z "$RTLA" ] && RTLA="./rtla" + [ -z "$RTLA" ] && RTLA="$(pwd)/rtla" [ -n "$TEST_COUNT" ] && echo "1..$TEST_COUNT" + + # Convert newlines to spaces so that seq can be used + unset subtests + if [ -n "$SUBTESTS" ]; then + subtests=$(echo "$SUBTESTS"|tr '\n' ' ') + fi } reset_osnoise() { @@ -50,8 +56,7 @@ # Simple check: run rtla with given arguments and test exit code. # If TEST_COUNT is set, run the test. Otherwise, just count. ctr=$(($ctr + 1)) - if [ -n "$TEST_COUNT" ] - then + if [[ -n "$TEST_COUNT" && ( -z "$subtests" || " $subtests " == *" $ctr "* ) ]]; then # Reset osnoise options before running test. [ "$NO_RESET_OSNOISE" == 1 ] || reset_osnoise @@ -62,6 +67,7 @@ # Run rtla; in case of failure, include its output as comment # in the test results. result=$(eval stdbuf -oL $TIMEOUT "$RTLA" $2 2>&1); exitcode=$? + popd >/dev/null failbuf='' fail=0 @@ -86,13 +92,22 @@ else # Add rtla output and exit code as comments in case of failure echo "not ok $ctr - $1" + echo "# command: $RTLA" $2 echo -n "$failbuf" echo "$result" | col -b | while read line; do echo "# $line"; done printf "#\n# exit code %s\n" $exitcode + + if [ -n "$STOPONFAIL" ]; then + # If the directory is not empty, tell the user where + # to find the output + if ! rmdir $tmpdir; then + echo "# output dir: $tmpdir" + fi + exit 1 + fi fi # Remove temporary directory - popd >/dev/null rm -r $tmpdir fi }
diff --git a/tools/tracing/rtla/tests/osnoise.t b/tools/tracing/rtla/tests/osnoise.t index 214fd40..352b302 100644 --- a/tools/tracing/rtla/tests/osnoise.t +++ b/tools/tracing/rtla/tests/osnoise.t
@@ -98,5 +98,7 @@ "osnoise TOOL -S 2 --on-threshold shell,command='if [ -f a ]; then echo 2; exit 1; else echo -n 1; touch a; fi' --on-threshold continue" 2 "^12$" "^2$" check_top_hist "with trace output at end" \ "osnoise TOOL -d 1s --on-end trace" 0 "^ Saving trace to osnoise_trace.txt$" +check_top_q_hist "action with no threshold" \ + "osnoise TOOL -d 1s --on-threshold shell,command='echo BAD'" 0 "" "BAD" test_end
diff --git a/tools/tracing/rtla/tests/timerlat.t b/tools/tracing/rtla/tests/timerlat.t index 5116857..1d317c8 100644 --- a/tools/tracing/rtla/tests/timerlat.t +++ b/tools/tracing/rtla/tests/timerlat.t
@@ -136,6 +136,8 @@ "timerlat TOOL -T 2 --on-threshold shell,command='if [ -f a ]; then echo 2; exit 1; else echo -n 1; touch a; fi' --on-threshold continue" 2 "^12$" "^2$" check_top_hist "with trace output at end" \ "timerlat TOOL -d 1s --on-end trace" 0 "^ Saving trace to timerlat_trace.txt$" +check_top_q_hist "action with no threshold" \ + "timerlat TOOL -d 1s --on-threshold shell,command='echo BAD'" 0 "" "BAD" # BPF action program tests if [ "$option" -eq 0 ]
diff --git a/tools/tracing/rtla/tests/unit/utils.c b/tools/tracing/rtla/tests/unit/utils.c index ce53cab..cdd84ff 100644 --- a/tools/tracing/rtla/tests/unit/utils.c +++ b/tools/tracing/rtla/tests/unit/utils.c
@@ -34,6 +34,75 @@ START_TEST(test_strtoi) } END_TEST +struct cpu_list_iterate_cb_data { + int index; + int *values; +}; + +static int cpu_list_iterate_callback(int cpu, void *data) +{ + struct cpu_list_iterate_cb_data *cb_data = data; + + ck_assert_int_eq(cpu, cb_data->values[cb_data->index++]); + + return 0; +} + +static int cpu_list_iterate_callback_error(int cpu, void *data) +{ + struct cpu_list_iterate_cb_data *cb_data = data; + + if (cpu > 10) + return -42; + + ck_assert_int_eq(cpu, cb_data->values[cb_data->index++]); + + return 0; +} + +START_TEST(test_cpu_list_iterate) +{ + struct cpu_list_iterate_cb_data cb_data; + int test_data_1[] = {1, 2, 3, 4}; + int test_data_2[] = {1, 2, 10, 11, 12}; + + cb_data.index = 0; + + cb_data.values = test_data_1; + ck_assert_int_eq(cpu_list_iterate("1,2,3,4", cpu_list_iterate_callback, &cb_data), 4); + ck_assert_int_eq(cb_data.index, 4); + cb_data.index = 0; + ck_assert_int_eq(cpu_list_iterate("1-4", cpu_list_iterate_callback, &cb_data), 4); + ck_assert_int_eq(cb_data.index, 4); + cb_data.index = 0; + ck_assert_int_eq(cpu_list_iterate("1,2-3,4", cpu_list_iterate_callback, &cb_data), 4); + ck_assert_int_eq(cb_data.index, 4); + cb_data.index = 0; + ck_assert_int_eq(cpu_list_iterate("1-3,4", cpu_list_iterate_callback, &cb_data), 4); + ck_assert_int_eq(cb_data.index, 4); + cb_data.index = 0; + ck_assert_int_eq(cpu_list_iterate("1,2-4", cpu_list_iterate_callback, &cb_data), 4); + ck_assert_int_eq(cb_data.index, 4); + + cb_data.index = 0; + ck_assert_int_eq(cpu_list_iterate("1,2-4", cpu_list_iterate_callback_error, &cb_data), 4); + ck_assert_int_eq(cb_data.index, 4); + cb_data.index = 0; + cb_data.values = test_data_2; + ck_assert_int_eq(cpu_list_iterate("1,2,10-12", cpu_list_iterate_callback_error, &cb_data), + -42); + ck_assert_int_eq(cb_data.index, 3); +} +END_TEST + +START_TEST(test_get_max_cpu_from_list) +{ + ck_assert_int_eq(get_max_cpu_from_list("1,2,3,4-60"), 60); + ck_assert_int_eq(get_max_cpu_from_list("2,1-7,4"), 7); + ck_assert_int_eq(get_max_cpu_from_list("invalid"), -1); +} +END_TEST + START_TEST(test_parse_cpu_set) { cpu_set_t set; @@ -98,6 +167,8 @@ Suite *utils_suite(void) TCase *tc = tcase_create("core"); tcase_add_test(tc, test_strtoi); + tcase_add_test(tc, test_cpu_list_iterate); + tcase_add_test(tc, test_get_max_cpu_from_list); tcase_add_test(tc, test_parse_cpu_set); tcase_add_test(tc, test_parse_prio);
diff --git a/tools/virtio/linux/device-id/virtio.h b/tools/virtio/linux/device-id/virtio.h new file mode 100644 index 0000000..3c3e8c5 --- /dev/null +++ b/tools/virtio/linux/device-id/virtio.h
@@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <../../include/linux/device-id/virtio.h>
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index c3c0ee25..99c9969 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig
@@ -40,9 +40,6 @@ bool depends on HAVE_KVM_DIRTY_RING -config KVM_MMIO - bool - config KVM_ASYNC_PF bool @@ -50,21 +47,12 @@ config KVM_ASYNC_PF_SYNC bool -config HAVE_KVM_MSI - bool - -config HAVE_KVM_READONLY_MEM - bool - config HAVE_KVM_CPU_RELAX_INTERCEPT bool config KVM_VFIO bool -config HAVE_KVM_INVALID_WAKEUPS - bool - config KVM_GENERIC_DIRTYLOG_READ_PROTECT bool @@ -79,12 +67,6 @@ tristate select IRQ_BYPASS_MANAGER -config HAVE_KVM_VCPU_RUN_PID_CHANGE - bool - -config HAVE_KVM_NO_POLL - bool - config VIRT_XFER_TO_GUEST_WORK bool @@ -100,9 +82,6 @@ config KVM_MMU_LOCKLESS_AGING bool -config KVM_GENERIC_MEMORY_ATTRIBUTES - bool - config KVM_GUEST_MEMFD select XARRAY_MULTI bool @@ -122,3 +101,6 @@ config HAVE_KVM_ARCH_GMEM_POPULATE bool depends on KVM_GUEST_MEMFD + +config KVM_VM_MEMORY_ATTRIBUTES + bool
diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm index d047d4c..ec7da59 100644 --- a/virt/kvm/Makefile.kvm +++ b/virt/kvm/Makefile.kvm
@@ -5,9 +5,14 @@ KVM ?= ../../../virt/kvm -kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o +KVM_DEV_NAME ?= kvm +KVM_DEV_MINOR ?= KVM_MINOR + +ccflags-y +=-DKVM_DEV_NAME=$(KVM_DEV_NAME) -DKVM_DEV_MINOR=$(KVM_DEV_MINOR) + +kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o \ + $(KVM)/coalesced_mmio.o kvm-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o -kvm-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o
diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index b8aaa96..6b3b635 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c
@@ -21,7 +21,10 @@ static struct kmem_cache *async_pf_cache; int kvm_async_pf_init(void) { - async_pf_cache = KMEM_CACHE(kvm_async_pf, 0); + async_pf_cache = kmem_cache_create(KVM_DEV_STRING(async_pf), + sizeof(struct kvm_async_pf), + __alignof__(struct kvm_async_pf), + 0, NULL); if (!async_pf_cache) return -ENOMEM;
diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index 6b1d901..3b54300 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c
@@ -15,6 +15,8 @@ #include <linux/slab.h> #include <linux/kvm.h> +#ifndef __KVM_NO_MMIO + #include "coalesced_mmio.h" static inline struct kvm_coalesced_mmio_dev *to_mmio(struct kvm_io_device *dev) @@ -188,3 +190,5 @@ int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm, */ return 0; } + +#endif /* !__KVM_NO_MMIO */
diff --git a/virt/kvm/coalesced_mmio.h b/virt/kvm/coalesced_mmio.h index 36f8426..77b811d 100644 --- a/virt/kvm/coalesced_mmio.h +++ b/virt/kvm/coalesced_mmio.h
@@ -11,7 +11,7 @@ * */ -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO #include <linux/list.h>
diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 93ad2eb..f43b8b2 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c
@@ -711,7 +711,8 @@ bool kvm_notify_irqfd_resampler(struct kvm *kvm, */ int kvm_irqfd_init(void) { - irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", WQ_PERCPU, 0); + irqfd_cleanup_wq = alloc_workqueue(__stringify(KVM_DEV_NAME) "-irqfd-cleanup", + WQ_PERCPU, 0); if (!irqfd_cleanup_wq) return -ENOMEM;
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 5c084d5..8c97a69 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c
@@ -4,9 +4,11 @@ #include <linux/falloc.h> #include <linux/fs.h> #include <linux/kvm_host.h> +#include <linux/maple_tree.h> #include <linux/mempolicy.h> #include <linux/pseudo_fs.h> #include <linux/pagemap.h> +#include <linux/swap.h> #include "kvm_mm.h" #include "guest_memfd.h" @@ -34,6 +36,13 @@ struct gmem_inode { struct list_head gmem_file_list; u64 flags; + /* + * Every index in this inode, whether memory is populated or + * not, is tracked in attributes. The entire range of indices, + * corresponding to the size of this inode, is represented in + * this maple tree. + */ + struct maple_tree attributes; }; static __always_inline struct gmem_inode *GMEM_I(struct inode *inode) @@ -61,9 +70,28 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn) return gfn - slot->base_gfn + slot->gmem.pgoff; } +static u64 kvm_gmem_get_default_attributes(struct inode *inode) +{ + bool init_shared = GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED; + + return init_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + +static u64 kvm_gmem_get_attributes(struct inode *inode, void *entry) +{ + if (WARN_ON_ONCE(!entry)) + return kvm_gmem_get_default_attributes(inode); + + return xa_to_value(entry); +} + static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index) { - return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED); + struct maple_tree *mt = &GMEM_I(inode)->attributes; + void *entry = mtree_load(mt, index); + + return kvm_gmem_get_attributes(inode, entry) & + KVM_MEMORY_ATTRIBUTE_PRIVATE; } static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) @@ -71,6 +99,24 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) return !kvm_gmem_is_private_mem(inode, index); } +static bool kvm_gmem_range_has_attributes(struct inode *inode, + pgoff_t index, size_t nr_pages, + u64 attributes) +{ + struct maple_tree *mt = &GMEM_I(inode)->attributes; + pgoff_t end = index + nr_pages - 1; + void *entry; + + lockdep_assert(mt_lock_is_held(mt)); + + mt_for_each(mt, entry, index, end) { + if (kvm_gmem_get_attributes(inode, entry) != attributes) + return false; + } + + return true; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -112,8 +158,11 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) return folio; } -static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode) +static enum kvm_gfn_range_filter kvm_gmem_get_all_gfns_filter(struct inode *inode) { + if (gmem_in_place_conversion) + return KVM_FILTER_SHARED | KVM_FILTER_PRIVATE; + if (GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED) return KVM_FILTER_SHARED; @@ -162,15 +211,13 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, + enum kvm_gfn_range_filter filter) { - enum kvm_gfn_range_filter attr_filter; struct gmem_file *f; - attr_filter = kvm_gmem_get_invalidate_filter(inode); - kvm_gmem_for_each_file(f, inode) - __kvm_gmem_invalidate_start(f, start, end, attr_filter); + __kvm_gmem_invalidate_start(f, start, end, filter); } static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start, @@ -196,6 +243,7 @@ static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start, static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { + enum kvm_gfn_range_filter filter = kvm_gmem_get_all_gfns_filter(inode); pgoff_t start = offset >> PAGE_SHIFT; pgoff_t end = (offset + len) >> PAGE_SHIFT; @@ -205,7 +253,7 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) */ filemap_invalidate_lock(inode->i_mapping); - kvm_gmem_invalidate_start(inode, start, end); + kvm_gmem_invalidate_start(inode, start, end, filter); truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); @@ -300,17 +348,56 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * dereferencing the slot for existing bindings needs to be protected * against memslot updates, specifically so that unbind doesn't race * and free the memslot (kvm_gmem_get_file() will return NULL). - * - * Since .release is called only when the reference count is zero, - * after which file_ref_get() and get_file_active() fail, - * kvm_gmem_get_pfn() cannot be using the file concurrently. - * file_ref_put() provides a full barrier, and get_file_active() the - * matching acquire barrier. */ mutex_lock(&kvm->slots_lock); filemap_invalidate_lock(inode->i_mapping); + /* + * Note! synchronize_srcu() is _not_ needed after nullifying memslot + * bindings as slot->gmem.file cannot be set back to a non-null value + * without the memslot first being deleted. I.e. this relies on the + * synchronize_srcu_expedited() in kvm_swap_active_memslots() to ensure + * kvm_gmem_get_pfn() (which runs with kvm->srcu held for read) can't + * grab a reference to slot->gmem.file even if the struct file object + * is reallocated (for use as a different file). + * + * file_ref_put() provides a full barrier, and __get_file_rcu() the + * matching acquire barrier, to ensure that kvm_gmem_get_file() (via + * __get_file_rcu()) sees refcount==0 or fails the "file reloaded" + * check (file != NULL due to nullifying the file pointer here). + * + * Unlike most other users of get_file_rcu(), where callers don't care + * if they race with a write, only that they have a reference to _a_ + * live file, kvm_gmem_get_pfn() needs to get the exact file that is + * associated with the memslot. Without the aforementioned SRCU + * synchronization, the following could happen: + * + * CPU0 CPU1 + * kvm_gmem_get_pfn() + * slot->gmem.file == A + * kvm_gmem_release()) + * slot->gmem.file = NULL + * + * kvm_set_memory_region() + * slot deleted + * + * kvm_set_memory_region() + * slot created + * slot->gmem.file = B (alloc the same object) + * + * get_file_active() + * file = B + * file_reloaded = B + * + * <KVM does weird things with an old memslot + new file> + * + * Obviously KVM would be broken in many places if the synchronization + * were omitted, but it's important to note that get_file_active() does + * NOT guarantee a reference to the "original" file was obtained, only + * that it grabbed a reference for the returned file, i.e. didn't grab + * a reference for file A, but then returned a pointer to file B. + */ xa_for_each(&f->bindings, index, slot) WRITE_ONCE(slot->gmem.file, NULL); @@ -320,7 +407,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * memory, as its lifetime is associated with the inode, not the file. */ __kvm_gmem_invalidate_start(f, 0, -1ul, - kvm_gmem_get_invalidate_filter(inode)); + kvm_gmem_get_all_gfns_filter(inode)); __kvm_gmem_invalidate_end(f, 0, -1ul); list_del(&f->entry); @@ -364,10 +451,13 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) return VM_FAULT_SIGBUS; - if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff)) - return VM_FAULT_SIGBUS; + filemap_invalidate_lock_shared(inode->i_mapping); + if (kvm_gmem_is_shared_mem(inode, vmf->pgoff)) + folio = kvm_gmem_get_folio(inode, vmf->pgoff); + else + folio = ERR_PTR(-EACCES); + filemap_invalidate_unlock_shared(inode->i_mapping); - folio = kvm_gmem_get_folio(inode, vmf->pgoff); if (IS_ERR(folio)) { if (PTR_ERR(folio) == -EAGAIN) return VM_FAULT_RETRY; @@ -448,11 +538,280 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); + struct inode *inode; + + if (!slot) + return 0; + + CLASS(gmem_get_file, file)(slot); + if (!file) + return 0; + + inode = file_inode(file); + + /* + * Rely on the maple tree's internal RCU lock to ensure a stable result. + * This result can become stale as soon as the lock is dropped, so the + * caller _must_ protect consumption of private vs. shared either by + * holding guest_memfd's invalidate lock for the entire duration, or by + * checking mmu_invalidate_retry_gfn() under mmu_lock to serialize + * against concurrent attribute updates. + */ + return kvm_gmem_is_private_mem(inode, kvm_gmem_get_index(slot, gfn)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private_gfn); + +/* + * Preallocate memory for attributes to be stored on a maple tree, pointed to + * by mas. Adjacent ranges with attributes identical to the new attributes + * will be merged. Also sets mas's bounds up for storing attributes. + * + * This maintains the invariant that ranges with the same attributes will + * always be merged. + */ +static int kvm_gmem_mas_preallocate(struct ma_state *mas, u64 attributes, + pgoff_t start, size_t nr_pages) +{ + pgoff_t end = start + nr_pages; + pgoff_t last = end - 1; + void *entry; + + /* Try extending range. entry is NULL on overflow/wrap-around. */ + mas_set(mas, end); + entry = mas_find(mas, end); + if (entry && xa_to_value(entry) == attributes) + last = mas->last; + + if (start > 0) { + mas_set(mas, start - 1); + entry = mas_find(mas, start - 1); + if (entry && xa_to_value(entry) == attributes) + start = mas->index; + } + + mas_set_range(mas, start, last); + return mas_preallocate(mas, xa_mk_value(attributes), GFP_KERNEL); +} + +static bool __folio_has_outstanding_references(struct folio *folio, + enum lru_cache_drained *drained) +{ + if (folio_maybe_dma_pinned(folio) || folio_mapped(folio)) + return true; + + /* 1 reference held by filemap_get_folios() in the folio batch. */ + lru_cache_drain_for_folio(folio, 1, drained); + + /* + * Outstanding references are anything other than those from the page + * cache, plus 1 temporary reference held by filemap_get_folios() in the + * folio batch. + */ + return folio_ref_count(folio) != folio_nr_pages(folio) + 1; +} + +static bool kvm_gmem_has_outstanding_references(struct inode *inode, + pgoff_t start, size_t nr_pages, + pgoff_t *err_index) +{ + enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED; + struct address_space *mapping = inode->i_mapping; + pgoff_t last = start + nr_pages - 1; + struct folio_batch fbatch; + pgoff_t next; + int i; + + folio_batch_init(&fbatch); + + next = start; + while (filemap_get_folios(mapping, &next, last, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + + if (__folio_has_outstanding_references(folio, &drained)) { + *err_index = max(start, folio->index); + folio_batch_release(&fbatch); + return true; + } + } + + folio_batch_release(&fbatch); + cond_resched(); + } + + return false; +} + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) +{ + struct folio_batch fbatch; + pgoff_t next = start; + int i; + + folio_batch_init(&fbatch); + while (filemap_get_folios(inode->i_mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + pgoff_t start_index, end_index; + kvm_pfn_t start_pfn; + kvm_pfn_t nr_pages; + + start_index = max(start, folio->index); + end_index = min(end, folio_next_index(folio)); + /* + * end_index is either in folio or points to + * the first page of the next folio. Hence, + * all pages in range [start_index, end_index) + * are contiguous. + */ + start_pfn = folio_file_pfn(folio, start_index); + nr_pages = end_index - start_index; + + kvm_arch_gmem_make_shared(start_pfn, nr_pages); + } + + folio_batch_release(&fbatch); + cond_resched(); + } +} +#else +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) {} +#endif + +static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, + size_t nr_pages, uint64_t attrs, + pgoff_t *err_index) +{ + bool to_private = attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE; + struct address_space *mapping = inode->i_mapping; + struct gmem_inode *gi = GMEM_I(inode); + enum kvm_gfn_range_filter filter; + pgoff_t end = start + nr_pages; + struct maple_tree *mt; + struct ma_state mas; + int r = 0; + + mt = &gi->attributes; + + filemap_invalidate_lock(mapping); + + if (kvm_gmem_range_has_attributes(inode, start, nr_pages, attrs)) + goto out; + + mas_init(&mas, mt, start); + r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); + if (r) { + *err_index = start; + goto out; + } + + if (to_private) { + /* + * Forcefully unmap the pages from all userspace page tables, + * and then verify there are no outstanding references, e.g. + * acquired via GUP or similar. Tell userspace to try again if + * there are outstanding references and hope that whatever has + * pinned the page will put its reference "soon". + */ + unmap_mapping_pages(mapping, start, nr_pages, false); + + if (kvm_gmem_has_outstanding_references(inode, start, nr_pages, + err_index)) { + mas_destroy(&mas); + r = -EAGAIN; + goto out; + } + } + + /* + * From this point on guest_memfd has performed necessary + * checks and can proceed to do guest-breaking changes. + */ + + filter = to_private ? KVM_FILTER_SHARED : KVM_FILTER_PRIVATE; + kvm_gmem_invalidate_start(inode, start, end, filter); + + if (!to_private && kvm_arch_has_gmem_convert()) + kvm_gmem_make_shared(inode, start, end); + + mas_store_prealloc(&mas, xa_mk_value(attrs)); + + kvm_gmem_invalidate_end(inode, start, end); +out: + filemap_invalidate_unlock(mapping); + return r; +} + +static long kvm_gmem_set_attributes(struct file *file, void __user *argp) +{ + struct gmem_file *f = file->private_data; + struct inode *inode = file_inode(file); + struct kvm_memory_attributes2 attrs; + pgoff_t err_index; + size_t nr_pages; + pgoff_t index; + int i, r; + + if (copy_from_user(&attrs, argp, sizeof(attrs))) + return -EFAULT; + + if (attrs.flags) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(attrs.reserved); i++) { + if (attrs.reserved[i]) + return -EINVAL; + } + if (!kvm_arch_has_private_mem(f->kvm)) + return -EINVAL; + if (attrs.attributes & ~KVM_MEMORY_ATTRIBUTE_PRIVATE) + return -EINVAL; + if (attrs.size == 0 || attrs.offset + attrs.size < attrs.offset) + return -EINVAL; + if (!PAGE_ALIGNED(attrs.offset) || !PAGE_ALIGNED(attrs.size)) + return -EINVAL; + + if (attrs.offset >= i_size_read(inode) || + attrs.offset + attrs.size > i_size_read(inode)) + return -EINVAL; + + nr_pages = attrs.size >> PAGE_SHIFT; + index = attrs.offset >> PAGE_SHIFT; + r = __kvm_gmem_set_attributes(inode, index, nr_pages, attrs.attributes, + &err_index); + if (r) { + attrs.error_offset = ((uint64_t)err_index) << PAGE_SHIFT; + + if (copy_to_user(argp, &attrs, sizeof(attrs))) + return -EFAULT; + } + + return r; +} + +static long kvm_gmem_ioctl(struct file *file, unsigned int ioctl, + unsigned long arg) +{ + switch (ioctl) { + case KVM_SET_MEMORY_ATTRIBUTES2: + if (!gmem_in_place_conversion) + return -ENOTTY; + + return kvm_gmem_set_attributes(file, (void __user *)arg); + default: + return -ENOTTY; + } +} + static struct file_operations kvm_gmem_fops = { .mmap = kvm_gmem_mmap, .open = generic_file_open, .release = kvm_gmem_release, .fallocate = kvm_gmem_fallocate, + .unlocked_ioctl = kvm_gmem_ioctl, }; static int kvm_gmem_migrate_folio(struct address_space *mapping, @@ -465,14 +824,27 @@ static int kvm_gmem_migrate_folio(struct address_space *mapping, static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio) { + struct inode *inode = mapping->host; + enum kvm_gfn_range_filter filter; pgoff_t start, end; - filemap_invalidate_lock_shared(mapping); + /* + * memory_failure() invokes ->error_remove_folio() while holding the + * poisoned folio lock. Do not block on mapping->invalidate_lock, as + * a writer that already holds the invalidate lock can be waiting for + * the same folio, e.g. truncation from MADV_REMOVE/FALLOC_FL_PUNCH_HOLE. + * + * If the invalidate lock is contended, fail the recovery instead of + * risking a deadlock in the memory-failure path. + */ + if (!filemap_invalidate_trylock_shared(mapping)) + return -EBUSY; start = folio->index; end = start + folio_nr_pages(folio); - kvm_gmem_invalidate_start(mapping->host, start, end); + filter = kvm_gmem_get_all_gfns_filter(inode); + kvm_gmem_invalidate_start(inode, start, end, filter); /* * Do not truncate the range, what action is taken in response to the @@ -520,6 +892,51 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return true; } +static int kvm_gmem_init_inode(struct inode *inode, loff_t size, u64 flags) +{ + struct gmem_inode *gi = GMEM_I(inode); + MA_STATE(mas, &gi->attributes, 0, (size >> PAGE_SHIFT) - 1); + u64 attrs; + int r; + + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + + /* + * guest_memfd memory is neither migratable nor swappable: set + * inaccessible to gate off both. + */ + mapping_set_inaccessible(inode->i_mapping); + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + gi->flags = flags; + + mt_set_external_lock(&gi->attributes, + &inode->i_mapping->invalidate_lock); + + /* + * Store default attributes for the entire gmem instance. Ensuring every + * index is represented in the maple tree at all times simplifies the + * conversion and merging logic. + */ + attrs = kvm_gmem_get_default_attributes(inode); + + /* + * Acquire the invalidation lock purely to make lockdep happy. The + * maple tree library expects all stores to be protected via the lock, + * and the library can't know when the tree is reachable only by the + * caller, as is the case here. + */ + filemap_invalidate_lock(inode->i_mapping); + r = mas_store_gfp(&mas, xa_mk_value(attrs), GFP_KERNEL); + filemap_invalidate_unlock(inode->i_mapping); + + return r; +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { static const char *name = "[kvm-gmem]"; @@ -550,16 +967,9 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) goto err_fops; } - inode->i_op = &kvm_gmem_iops; - inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mode |= S_IFREG; - inode->i_size = size; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); - mapping_set_inaccessible(inode->i_mapping); - /* Unmovable mappings are supposed to be marked unevictable as well. */ - WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); - - GMEM_I(inode)->flags = flags; + err = kvm_gmem_init_inode(inode, size, flags); + if (err) + goto err_inode; file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops); if (IS_ERR(file)) { @@ -603,19 +1013,21 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) return __kvm_gmem_create(kvm, size, flags); } -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) { uoff_t size = slot->npages << PAGE_SHIFT; - unsigned long start, end; struct gmem_file *f; struct inode *inode; struct file *file; - int r = -EINVAL; + BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset)); BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); + if (WARN_ON_ONCE(slot->flags & KVM_MEMSLOT_GMEM_ONLY)) + return -EINVAL; + file = fget(fd); if (!file) return -EBADF; @@ -632,84 +1044,93 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode)) goto err; - filemap_invalidate_lock(inode->i_mapping); - - start = offset >> PAGE_SHIFT; - end = start + slot->npages; - - if (!xa_empty(&f->bindings) && - xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { - r = -EEXIST; - filemap_invalidate_unlock(inode->i_mapping); - goto err; - } - /* * memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so * kvm_gmem_bind() must occur on a new memslot. Because the memslot * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file. */ - WRITE_ONCE(slot->gmem.file, file); - slot->gmem.pgoff = start; - if (kvm_gmem_supports_mmap(inode)) + slot->gmem.file = file; + slot->gmem.pgoff = offset >> PAGE_SHIFT; + if (gmem_in_place_conversion || kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; - xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL); - filemap_invalidate_unlock(inode->i_mapping); - /* - * Drop the reference to the file, even on success. The file pins KVM, - * not the other way 'round. Active bindings are invalidated if the - * file is closed before memslots are destroyed. + * Gift the caller a reference to the file. The reference will be + * dropped after bindings are established, or if installing the new + * memslot ultimately fails. */ - r = 0; + return 0; + err: fput(file); - return r; + return -EINVAL; } -static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f) +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot) { - unsigned long start = slot->gmem.pgoff; - unsigned long end = start + slot->npages; + struct gmem_file *f = slot->gmem.file->private_data; + struct inode *inode = file_inode(slot->gmem.file); + unsigned long start, end; + int r; - xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + if (WARN_ON_ONCE(slot->gmem.file->f_op != &kvm_gmem_fops)) + return -EIO; - /* - * synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn() - * cannot see this memslot. - */ - WRITE_ONCE(slot->gmem.file, NULL); + filemap_invalidate_lock(inode->i_mapping); + + start = slot->gmem.pgoff; + end = start + slot->npages; + + if (!xa_empty(&f->bindings) && + xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { + filemap_invalidate_unlock(inode->i_mapping); + return -EEXIST; + } + + r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL)); + if (r) + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + filemap_invalidate_unlock(inode->i_mapping); + + return r; } void kvm_gmem_unbind(struct kvm_memory_slot *slot) { + struct file *file = slot->gmem.file; + unsigned long start = slot->gmem.pgoff; + unsigned long end = start + slot->npages; + struct gmem_file *f; + /* * Nothing to do if the underlying file was _already_ closed, as * kvm_gmem_release() invalidates and nullifies all bindings. */ - if (!slot->gmem.file) + if (!file) return; - CLASS(gmem_get_file, file)(slot); - /* * However, if the file is _being_ closed, then the bindings need to be * removed as kvm_gmem_release() might not run until after the memslot - * is freed. Note, modifying the bindings is safe even though the file - * is dying as kvm_gmem_release() nullifies slot->gmem.file under - * slots_lock, and only puts its reference to KVM after destroying all - * bindings. I.e. reaching this point means kvm_gmem_release() hasn't - * yet destroyed the bindings or freed the gmem_file, and can't do so - * until the caller drops slots_lock. + * is freed. Modifying the bindings is safe even if the file is dying + * as kvm_gmem_release() nullifies slot->gmem.file under slots_lock, + * and only puts its reference to KVM after destroying all bindings. + * I.e. reaching this point means kvm_gmem_release() hasn't destroyed + * the bindings or freed the gmem_file and can't do so until the caller + * drops slots_lock, so there's no need to verify the file is live. */ - if (!file) { - __kvm_gmem_unbind(slot, slot->gmem.file->private_data); - return; - } + f = file->private_data; filemap_invalidate_lock(file->f_mapping); - __kvm_gmem_unbind(slot, file->private_data); + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + /* + * Note, the caller is responsible for ensuring the slot is unreachable + * before unbinding, e.g. by synchronizing SRCU after deleting the slot, + * to guarantee kvm_gmem_get_pfn() can't see the slot+file. + */ + WRITE_ONCE(slot->gmem.file, NULL); filemap_invalidate_unlock(file->f_mapping); } @@ -743,6 +1164,11 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, return ERR_PTR(-EHWPOISON); } + if (!folio_test_uptodate(folio)) { + clear_highpage(folio_page(folio, 0)); + folio_mark_uptodate(folio); + } + *pfn = folio_file_pfn(folio, index); if (max_order) *max_order = 0; @@ -751,8 +1177,7 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, } int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order) + gfn_t gfn, kvm_pfn_t *pfn, int *max_order) { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; @@ -764,28 +1189,24 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, if (!file) return -EFAULT; - folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); - if (IS_ERR(folio)) - return PTR_ERR(folio); + filemap_invalidate_lock_shared(file_inode(file)->i_mapping); - if (!folio_test_uptodate(folio)) { - clear_highpage(folio_page(folio, 0)); - folio_mark_uptodate(folio); + folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); + if (IS_ERR(folio)) { + r = PTR_ERR(folio); + goto out; } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - if (kvm_gmem_is_private_mem(file_inode(file), index)) + if (kvm_arch_has_gmem_convert() && + kvm_gmem_is_private_mem(file_inode(file), index)) r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, (kvm_pfn_t)1 << *max_order); -#endif folio_unlock(folio); + folio_put(folio); - if (!r) - *page = folio_file_page(folio, index); - else - folio_put(folio); - +out: + filemap_invalidate_unlock_shared(file_inode(file)->i_mapping); return r; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn); @@ -811,16 +1232,12 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, folio_unlock(folio); - if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, - KVM_MEMORY_ATTRIBUTE_PRIVATE, - KVM_MEMORY_ATTRIBUTE_PRIVATE)) { + if (!kvm_is_private_gfn(kvm, gfn)) { ret = -EINVAL; goto out_put_folio; } ret = post_populate(kvm, gfn, pfn, src_page, opaque); - if (!ret) - folio_mark_uptodate(folio); out_put_folio: folio_put(folio); @@ -917,6 +1334,15 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) mpol_shared_policy_init(&gi->policy, NULL); + /* + * Memory attributes are protected by the filemap invalidation lock, but + * the lock structure isn't available at this time. Immediately mark + * maple tree as using external locking so that accessing the tree + * before it's fully initialized results in NULL pointer dereferences + * and not more subtle bugs. + */ + mt_init_flags(&gi->attributes, MT_FLAGS_LOCK_EXTERN | MT_FLAGS_USE_RCU); + gi->flags = 0; INIT_LIST_HEAD(&gi->gmem_file_list); return &gi->vfs_inode; @@ -924,7 +1350,26 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) static void kvm_gmem_destroy_inode(struct inode *inode) { - mpol_free_shared_policy(&GMEM_I(inode)->policy); + struct gmem_inode *gi = GMEM_I(inode); + + mpol_free_shared_policy(&gi->policy); + + /* + * Note! Checking for an empty tree is functionally necessary + * to avoid explosions if the tree hasn't been fully + * initialized, i.e. if the inode is being destroyed before + * guest_memfd can set the external lock, lockdep would find + * that the tree's internal ma_lock was not held. + */ + if (!mtree_empty(&gi->attributes)) { + /* + * Acquire the invalidation lock purely to make lockdep happy, + * the inode is unreachable at this point. + */ + filemap_invalidate_lock(inode->i_mapping); + __mt_destroy(&gi->attributes); + filemap_invalidate_unlock(inode->i_mapping); + } } static void kvm_gmem_free_inode(struct inode *inode)
diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h index 0f9c6f8..01bd359 100644 --- a/virt/kvm/guest_memfd.h +++ b/virt/kvm/guest_memfd.h
@@ -8,8 +8,9 @@ int kvm_gmem_init(struct module *module); void kvm_gmem_exit(void); int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset); +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset); +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_gmem_unbind(struct kvm_memory_slot *slot); #else static inline int kvm_gmem_init(struct module *module) @@ -17,9 +18,17 @@ static inline int kvm_gmem_init(struct module *module) return 0; } static inline void kvm_gmem_exit(void) {}; -static inline int kvm_gmem_bind(struct kvm *kvm, - struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) + +static inline int kvm_gmem_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) +{ + WARN_ON_ONCE(1); + return -EIO; +} + +static inline int kvm_gmem_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot) { WARN_ON_ONCE(1); return -EIO;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5a78284..cf4cb58 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c
@@ -10,6 +10,8 @@ * Yaniv Kamay <yaniv@qumranet.com> */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <kvm/iodev.h> #include <linux/kvm_host.h> @@ -102,6 +104,12 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +bool __ro_after_init gmem_in_place_conversion = false; +module_param(gmem_in_place_conversion, bool, 0444); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); +#endif + /* * Ordering of locks: * @@ -470,7 +478,7 @@ static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_dirty_ring_free(&vcpu->dirty_ring); /* - * No need for rcu_read_lock as VCPU_RUN is the only place that changes + * No need for rcu_read_lock as KVM_RUN is the only place that changes * the vcpu->pid pointer, and at destruction time all file descriptors * are already gone. */ @@ -813,6 +821,16 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, /* Pairs with the increment in range_start(). */ spin_lock(&kvm->mn_invalidate_lock); + kvm->gpc_invalidate_seq++; + + /* + * As with the MMU sequence counter and mmu_invalidate_in_progress, the + * GPC sequence increase must be visible before the invalidate count + * goes to zero. Pairs with the smp_rmb() in + * mmu_notifier_retry_cache(). + */ + smp_wmb(); + if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count)) --kvm->mn_active_invalidate_count; wake = !kvm->mn_active_invalidate_count; @@ -1013,7 +1031,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) mutex_lock(&kvm_debugfs_lock); dent = debugfs_lookup(dir_name, kvm_debugfs_dir); if (dent) { - pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name); + pr_warn_ratelimited("debugfs: duplicate directory %s\n", dir_name); dput(dent); mutex_unlock(&kvm_debugfs_lock); return 0; @@ -1116,7 +1134,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) spin_lock_init(&kvm->mn_invalidate_lock); rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT); #endif @@ -1301,7 +1319,7 @@ static void kvm_destroy_vm(struct kvm *kvm) cleanup_srcu_struct(&kvm->irq_srcu); srcu_barrier(&kvm->srcu); cleanup_srcu_struct(&kvm->srcu); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_destroy(&kvm->mem_attr_array); #endif kvm_arch_free_vm(kvm); @@ -1314,7 +1332,7 @@ void kvm_get_kvm(struct kvm *kvm) { refcount_inc(&kvm->users_count); } -EXPORT_SYMBOL_GPL(kvm_get_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_kvm); /* * Make sure the vm is not during destruction, which is a safe version of @@ -1324,14 +1342,14 @@ bool kvm_get_kvm_safe(struct kvm *kvm) { return refcount_inc_not_zero(&kvm->users_count); } -EXPORT_SYMBOL_GPL(kvm_get_kvm_safe); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_kvm_safe); void kvm_put_kvm(struct kvm *kvm) { if (refcount_dec_and_test(&kvm->users_count)) kvm_destroy_vm(kvm); } -EXPORT_SYMBOL_GPL(kvm_put_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_put_kvm); /* * Used to put a reference that was taken on behalf of an object associated @@ -1352,6 +1370,8 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) kvm_irqfd_release(kvm); + WRITE_ONCE(kvm->file, NULL); + kvm_put_kvm(kvm); return 0; } @@ -1748,10 +1768,10 @@ static void kvm_commit_memory_region(struct kvm *kvm, kvm_destroy_dirty_bitmap(old); /* - * Unbind the guest_memfd instance as needed; the @new slot has - * already created its own binding. TODO: Drop the WARN when - * dirty logging guest_memfd memslots is supported. Until then, - * flags-only changes on guest_memfd slots should be impossible. + * TODO: Drop the WARN and do the unbind() call only for MOVE + * when dirty logging guest_memfd memslots is supported. Until + * then, flags-only changes on guest_memfd slots should also be + * impossible; unbind the old memslot for defense-in-depth. */ if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD)) kvm_gmem_unbind(old); @@ -1937,20 +1957,16 @@ static int kvm_set_memslot(struct kvm *kvm, } r = kvm_prepare_memory_region(kvm, old, new, change); - if (r) { - /* - * For DELETE/MOVE, revert the above INVALID change. No - * modifications required since the original slot was preserved - * in the inactive slots. Changing the active memslots also - * release slots_arch_lock. - */ - if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { - kvm_activate_memslot(kvm, invalid_slot, old); - kfree(invalid_slot); - } else { - mutex_unlock(&kvm->slots_arch_lock); + if (r) + goto err; + + if (change == KVM_MR_CREATE && (new->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_commit_memory_region(kvm, new); + if (r) { + kvm_arch_free_memslot(kvm, new); + kvm_destroy_dirty_bitmap(new); + goto err; } - return r; } /* @@ -1983,6 +1999,20 @@ static int kvm_set_memslot(struct kvm *kvm, kvm_commit_memory_region(kvm, old, new, change); return 0; + +err: + /* + * For DELETE/MOVE, revert the above INVALID change. No modifications + * required since the original slot was preserved in the inactive slots. + * Changing the active memslots also release slots_arch_lock. + */ + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { + kvm_activate_memslot(kvm, invalid_slot, old); + kfree(invalid_slot); + } else { + mutex_unlock(&kvm->slots_arch_lock); + } + return r; } static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id, @@ -2111,21 +2141,30 @@ static int kvm_set_memory_region(struct kvm *kvm, new->npages = npages; new->flags = mem->flags; new->userspace_addr = mem->userspace_addr; - if (mem->flags & KVM_MEM_GUEST_MEMFD) { - r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset); + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_prepare_memory_region(kvm, new, mem->guest_memfd, + mem->guest_memfd_offset); if (r) goto out; } r = kvm_set_memslot(kvm, old, new, change); + + /* + * Drop the reference to the gmem file, even on success. The file pins + * KVM, not the other way 'round. Active bindings are invalidated if + * the file is closed before memslots are destroyed. + */ +#ifdef CONFIG_KVM_GUEST_MEMFD + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) + fput(new->gmem.file); +#endif + if (r) - goto out_unbind; + goto out; return 0; -out_unbind: - if (mem->flags & KVM_MEM_GUEST_MEMFD) - kvm_gmem_unbind(new); out: kfree(new); return r; @@ -2134,7 +2173,7 @@ static int kvm_set_memory_region(struct kvm *kvm, int kvm_set_internal_memslot(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem) { - if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS)) + if (WARN_ON_ONCE((u16)mem->slot < KVM_USER_MEM_SLOTS)) return -EINVAL; if (WARN_ON_ONCE(mem->flags)) @@ -2425,32 +2464,41 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, } #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static u64 kvm_supported_mem_attributes(struct kvm *kvm) +#ifdef kvm_arch_has_private_mem +static u64 kvm_supports_private_mem(struct kvm *kvm) { - if (!kvm || kvm_arch_has_private_mem(kvm)) - return KVM_MEMORY_ATTRIBUTE_PRIVATE; + return !kvm || kvm_arch_has_private_mem(kvm); +} +#else +#define kvm_supports_private_mem(kvm) false +#endif - return 0; +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) +{ + if (gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; } /* * Returns true if _all_ gfns in the range [@start, @end) have attributes * such that the bits in @mask match @attrs. */ -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs) +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs) { XA_STATE(xas, &kvm->mem_attr_array, start); unsigned long index; void *entry; - mask &= kvm_supported_mem_attributes(kvm); + mask &= kvm_supported_vm_mem_attributes(kvm); if (attrs & ~mask) return false; if (end == start + 1) - return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; + return (kvm_get_vm_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); @@ -2543,8 +2591,8 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, KVM_MMU_UNLOCK(kvm); } -static bool kvm_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +static bool kvm_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { /* * Unconditionally add the range to the invalidation set, regardless of @@ -2559,18 +2607,18 @@ static bool kvm_pre_set_memory_attributes(struct kvm *kvm, */ kvm_mmu_invalidate_range_add(kvm, range->start, range->end); - return kvm_arch_pre_set_memory_attributes(kvm, range); + return kvm_arch_pre_set_vm_memory_attributes(kvm, range); } /* Set @attributes for the gfn range [@start, @end). */ -static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, +static int kvm_set_vm_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attributes) { struct kvm_mmu_notifier_range pre_set_range = { .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_pre_set_memory_attributes, + .handler = kvm_pre_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_start, .flush_on_ret = true, .may_block = true, @@ -2579,7 +2627,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_arch_post_set_memory_attributes, + .handler = kvm_arch_post_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_end, .may_block = true, }; @@ -2589,12 +2637,12 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, entry = attributes ? xa_mk_value(attributes) : NULL; - trace_kvm_vm_set_mem_attributes(start, end, attributes); + trace_kvm_set_vm_mem_attributes(start, end, attributes); mutex_lock(&kvm->slots_lock); /* Nothing to do if the entire range has the desired attributes. */ - if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes)) + if (kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attributes)) goto out_unlock; /* @@ -2634,7 +2682,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, /* flags is currently not used. */ if (attrs->flags) return -EINVAL; - if (attrs->attributes & ~kvm_supported_mem_attributes(kvm)) + if (attrs->attributes & ~kvm_supported_vm_mem_attributes(kvm)) return -EINVAL; if (attrs->size == 0 || attrs->address + attrs->size < attrs->address) return -EINVAL; @@ -2651,9 +2699,29 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, */ BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long)); - return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes); + return kvm_set_vm_mem_attributes(kvm, start, end, attrs->attributes); } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ + +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +DEFINE_STATIC_CALL_RET0(__kvm_is_private_gfn, kvm_is_private_gfn_t); +EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn); + +static bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + +static void kvm_init_memory_attributes(void) +{ + if (gmem_in_place_conversion) + static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn); + else + static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); +} +#else +static void kvm_init_memory_attributes(void) { } +#endif struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { @@ -4088,7 +4156,7 @@ static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf) else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) page = virt_to_page(vcpu->arch.pio_data); #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); #endif @@ -4142,9 +4210,9 @@ static struct file_operations kvm_vcpu_fops = { */ static int create_vcpu_fd(struct kvm_vcpu *vcpu) { - char name[8 + 1 + ITOA_MAX_LEN + 1]; + char name[sizeof(__stringify(KVM_DEV_NAME) "-vcpu:") + ITOA_MAX_LEN + 1]; - snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); + snprintf(name, sizeof(name), __stringify(KVM_DEV_NAME) "-vcpu:%d", vcpu->vcpu_id); return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); } @@ -4338,9 +4406,9 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) { int fd; struct file *file; - char name[15 + ITOA_MAX_LEN + 1]; + char name[sizeof(__stringify(KVM_DEV_NAME) "-vcpu-stats:") + ITOA_MAX_LEN + 1]; - snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id); + snprintf(name, sizeof(name), __stringify(KVM_DEV_NAME) "-vcpu-stats:%d", vcpu->vcpu_id); fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) @@ -4363,7 +4431,7 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { - int idx; + int idx, err; long r; u64 full_size; @@ -4375,6 +4443,14 @@ static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, range->gpa + range->size <= range->gpa) return -EINVAL; + /* + * Certain architectures (e.g. arm64) need to reject the ioctl 'early' + * before vcpu_load(). + */ + err = kvm_arch_pre_fault_allowed(vcpu); + if (err) + return err; + vcpu_load(vcpu); idx = srcu_read_lock(&vcpu->kvm->srcu); @@ -4899,7 +4975,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: case KVM_CAP_INTERNAL_ERROR_DATA: -#ifdef CONFIG_HAVE_KVM_MSI +#ifdef __KVM_HAVE_MSI case KVM_CAP_SIGNAL_MSI: #endif #ifdef CONFIG_HAVE_KVM_IRQCHIP @@ -4910,7 +4986,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_ENABLE_CAP_VM: case KVM_CAP_HALT_POLL: return 1; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO case KVM_CAP_COALESCED_MMIO: return KVM_COALESCED_MMIO_PAGE_OFFSET; case KVM_CAP_COALESCED_PIO: @@ -4951,15 +5027,20 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_SYSTEM_EVENT_DATA: case KVM_CAP_DEVICE_CTRL: return 1; -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: - return kvm_supported_mem_attributes(kvm); + return kvm_supported_vm_mem_attributes(kvm); #endif #ifdef CONFIG_KVM_GUEST_MEMFD case KVM_CAP_GUEST_MEMFD: return 1; case KVM_CAP_GUEST_MEMFD_FLAGS: return kvm_gmem_get_supported_flags(kvm); + case KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES: + if (!gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; #endif default: break; @@ -5153,7 +5234,7 @@ static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) if (fd < 0) return fd; - file = anon_inode_getfile_fmode("kvm-vm-stats", + file = anon_inode_getfile_fmode(__stringify(KVM_DEV_NAME) "-vm-stats", &kvm_vm_stats_fops, kvm, O_RDONLY, FMODE_PREAD); if (IS_ERR(file)) { put_unused_fd(fd); @@ -5251,7 +5332,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO case KVM_REGISTER_COALESCED_MMIO: { struct kvm_coalesced_mmio_zone zone; @@ -5289,7 +5370,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_ioeventfd(kvm, &data); break; } -#ifdef CONFIG_HAVE_KVM_MSI +#ifdef __KVM_HAVE_MSI case KVM_SIGNAL_MSI: { struct kvm_msi msi; @@ -5355,7 +5436,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_SET_MEMORY_ATTRIBUTES: { struct kvm_memory_attributes attrs; @@ -5366,7 +5447,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs); break; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ case KVM_CREATE_DEVICE: { struct kvm_create_device cd; @@ -5500,11 +5581,16 @@ static struct file_operations kvm_vm_fops = { KVM_COMPAT(kvm_vm_compat_ioctl), }; -bool file_is_kvm(struct file *file) +#ifdef kvm_file_to_kvm_arch +struct kvm *kvm_file_to_kvm_fn(struct file *file) { - return file && file->f_op == &kvm_vm_fops; + if (!file || file->f_op != &kvm_vm_fops) + return NULL; + + return file->private_data; } -EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm); +EXPORT_SYMBOL_GPL(kvm_file_to_kvm_fn); +#endif static int kvm_dev_ioctl_create_vm(unsigned long type) { @@ -5525,12 +5611,14 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) goto put_fd; } - file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); + file = anon_inode_getfile(__stringify(KVM_DEV_NAME) "-vm", &kvm_vm_fops, kvm, O_RDWR); if (IS_ERR(file)) { r = PTR_ERR(file); goto put_kvm; } + kvm->file = file; + /* * Don't call kvm_put_kvm anymore at this point; file->f_op is * already set, with ->release() being kvm_vm_release(). In error @@ -5573,7 +5661,7 @@ static long kvm_dev_ioctl(struct file *filp, #ifdef CONFIG_X86 r += PAGE_SIZE; /* pio data page */ #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO r += PAGE_SIZE; /* coalesced mmio ring page */ #endif break; @@ -5591,8 +5679,8 @@ static struct file_operations kvm_chardev_ops = { }; static struct miscdevice kvm_dev = { - KVM_MINOR, - "kvm", + KVM_DEV_MINOR, + __stringify(KVM_DEV_NAME), &kvm_chardev_ops, }; @@ -5626,7 +5714,7 @@ static int kvm_enable_virtualization_cpu(void) return 0; if (kvm_arch_enable_virtualization_cpu()) { - pr_info("kvm: enabling virtualization on CPU%d failed\n", + pr_info("enabling virtualization on CPU%d failed\n", raw_smp_processor_id()); return -EIO; } @@ -5673,7 +5761,7 @@ static void kvm_shutdown(void *data) * is used for both shutdown and reboot scenarios, i.e. neither name is * 100% comprehensive. */ - pr_info("kvm: exiting hardware virtualization\n"); + pr_info("exiting hardware virtualization\n"); on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1); } @@ -6084,7 +6172,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, * attached devices. Otherwise, destroy the caller's device only. */ if (!new_bus) { - pr_err("kvm: failed to shrink bus, removing it completely\n"); + pr_err("failed to shrink bus, removing it completely\n"); kvm_io_bus_destroy(bus); return -ENOMEM; } @@ -6371,7 +6459,7 @@ static void kvm_init_debug(void) const struct kvm_stats_desc *pdesc; int i; - kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); + kvm_debugfs_dir = debugfs_create_dir(__stringify(KVM_DEV_NAME), NULL); for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { pdesc = &kvm_vm_stats_desc[i]; @@ -6516,7 +6604,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) if (!vcpu_align) vcpu_align = __alignof__(struct kvm_vcpu); kvm_vcpu_cache = - kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, + kmem_cache_create_usercopy(KVM_DEV_STRING(vcpu), vcpu_size, vcpu_align, SLAB_ACCOUNT, offsetof(struct kvm_vcpu, arch), offsetofend(struct kvm_vcpu, stats_id) @@ -6549,6 +6637,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; + kvm_init_memory_attributes(); kvm_init_debug(); r = kvm_vfio_ops_init(); @@ -6569,7 +6658,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) */ r = misc_register(&kvm_dev); if (r) { - pr_err("kvm: misc device register failed\n"); + pr_err("misc device register failed\n"); goto err_register; }
diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 728d2c1..3659686b 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c
@@ -124,7 +124,7 @@ static void gpc_unmap(kvm_pfn_t pfn, void *khva) #endif } -static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_seq) +static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long gpc_seq) { /* * mn_active_invalidate_count acts for all intents and purposes @@ -136,20 +136,20 @@ static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_s * Note, it does not matter that mn_active_invalidate_count * is not protected by gpc->lock. It is guaranteed to * be elevated before the mmu_notifier acquires gpc->lock, and - * isn't dropped until after mmu_invalidate_seq is updated. + * isn't dropped until after gpc_invalidate_seq is updated. */ if (kvm->mn_active_invalidate_count) return true; /* * Ensure mn_active_invalidate_count is read before - * mmu_invalidate_seq. This pairs with the smp_wmb() in - * mmu_notifier_invalidate_range_end() to guarantee either the + * gpc_invalidate_seq. This pairs with the smp_wmb() in + * kvm_mmu_notifier_invalidate_range_end() to guarantee either the * old (non-zero) value of mn_active_invalidate_count or the - * new (incremented) value of mmu_invalidate_seq is observed. + * new (incremented) value of gpc_invalidate_seq is observed. */ smp_rmb(); - return kvm->mmu_invalidate_seq != mmu_seq; + return kvm->gpc_invalidate_seq != gpc_seq; } static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) @@ -158,7 +158,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) void *old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva); kvm_pfn_t new_pfn = KVM_PFN_ERR_FAULT; void *new_khva = NULL; - unsigned long mmu_seq; + unsigned long gpc_seq; struct page *page; struct kvm_follow_pfn kfp = { @@ -181,7 +181,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) gpc->valid = false; do { - mmu_seq = gpc->kvm->mmu_invalidate_seq; + gpc_seq = gpc->kvm->gpc_invalidate_seq; smp_rmb(); write_unlock_irq(&gpc->lock); @@ -232,7 +232,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) * attempting to refresh. */ WARN_ON_ONCE(gpc->valid); - } while (mmu_notifier_retry_cache(gpc->kvm, mmu_seq)); + } while (mmu_notifier_retry_cache(gpc->kvm, gpc_seq)); gpc->valid = true; gpc->pfn = new_pfn;
diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index 6cdc4e9..19548a4 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c
@@ -35,15 +35,15 @@ struct kvm_vfio { bool noncoherent; }; -static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm) +static void kvm_vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) { - void (*fn)(struct file *file, struct kvm *kvm); + void (*fn)(struct file *vfio_file, struct file *kvm); fn = symbol_get(vfio_file_set_kvm); if (!fn) return; - fn(file, kvm); + fn(vfio_file, kvm); symbol_put(vfio_file_set_kvm); } @@ -144,6 +144,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) { struct kvm_vfio *kv = dev->private; struct kvm_vfio_file *kvf; + struct file *kvm_file __free(fput) = NULL; struct file *filp __free(fput) = NULL; filp = fget(fd); @@ -154,6 +155,10 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) if (!kvm_vfio_file_is_valid(filp)) return -EINVAL; + kvm_file = get_file_active(&dev->kvm->file); + if (!kvm_file) + return -ENOENT; + guard(mutex)(&kv->lock); list_for_each_entry(kvf, &kv->file_list, node) { @@ -168,7 +173,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvf->file = get_file(filp); list_add_tail(&kvf->node, &kv->file_list); - kvm_vfio_file_set_kvm(kvf->file, dev->kvm); + kvm_vfio_file_set_kvm(kvf->file, kvm_file); kvm_vfio_update_coherency(dev); return 0;